001/*
002 * Licensed to the Apache Software Foundation (ASF) under one or more
003 * contributor license agreements.  See the NOTICE file distributed with
004 * this work for additional information regarding copyright ownership.
005 * The ASF licenses this file to You under the Apache License, Version 2.0
006 * (the "License"); you may not use this file except in compliance with
007 * the License.  You may obtain a copy of the License at
008 *
009 *      http://www.apache.org/licenses/LICENSE-2.0
010 *
011 * Unless required by applicable law or agreed to in writing, software
012 * distributed under the License is distributed on an "AS IS" BASIS,
013 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
014 * See the License for the specific language governing permissions and
015 * limitations under the License.
016 */
017
018package org.apache.commons.net.ftp.parser;
019
020import java.text.ParseException;
021import java.util.List;
022
023import org.apache.commons.net.ftp.FTPClientConfig;
024import org.apache.commons.net.ftp.FTPFile;
025
026/**
027 * Implementation of FTPFileEntryParser and FTPFileListParser for IBM zOS/MVS Systems.
028 *
029 * @see org.apache.commons.net.ftp.FTPFileEntryParser FTPFileEntryParser (for usage instructions)
030 */
031public class MVSFTPEntryParser extends ConfigurableFTPFileEntryParserImpl {
032
033    static final int UNKNOWN_LIST_TYPE = -1;
034    static final int FILE_LIST_TYPE = 0;
035    static final int MEMBER_LIST_TYPE = 1;
036    static final int UNIX_LIST_TYPE = 2;
037    static final int JES_LEVEL_1_LIST_TYPE = 3;
038    static final int JES_LEVEL_2_LIST_TYPE = 4;
039
040    /**
041     * Dates are ignored for file lists, but are used for member lists where possible
042     */
043    static final String DEFAULT_DATE_FORMAT = "yyyy/MM/dd HH:mm"; // 2001/09/18
044                                                                  // 13:52
045
046    /**
047     * Matches these entries:
048     *
049     * <pre>
050     *  Volume Unit    Referred Ext Used Recfm Lrecl BlkSz Dsorg Dsname
051     *  B10142 3390   2006/03/20  2   31  F       80    80  PS   MDI.OKL.WORK
052     * </pre>
053     *
054     * @see <a href= "https://www.ibm.com/support/knowledgecenter/zosbasics/com.ibm.zos.zconcepts/zconcepts_159.htm">Data set record formats</a>
055     */
056    static final String FILE_LIST_REGEX = "\\S+\\s+" + // volume
057                                                       // ignored
058            "\\S+\\s+" + // unit - ignored
059            "\\S+\\s+" + // access date - ignored
060            "\\S+\\s+" + // extents -ignored
061            // If the values are too large, the fields may be merged (NET-639)
062            "(?:\\S+\\s+)?" + // used - ignored
063            "(?:F|FB|V|VB|U)\\s+" + // recfm - F[B], V[B], U
064            "\\S+\\s+" + // logical record length -ignored
065            "\\S+\\s+" + // block size - ignored
066            "(PS|PO|PO-E)\\s+" + // Dataset organisation. Many exist
067            // but only support: PS, PO, PO-E
068            "(\\S+)\\s*"; // Dataset Name (file name)
069
070    /**
071     * Matches these entries:
072     *
073     * <pre>
074     *   Name      VV.MM   Created       Changed      Size  Init   Mod   Id
075     *   TBSHELF   01.03 2002/09/12 2002/10/11 09:37    11    11     0 KIL001
076     * </pre>
077     */
078    static final String MEMBER_LIST_REGEX = "(\\S+)\\s+" + // name
079            "\\S+\\s+" + // version, modification (ignored)
080            "\\S+\\s+" + // create date (ignored)
081            "(\\S+)\\s+" + // modification date
082            "(\\S+)\\s+" + // modification time
083            "\\S+\\s+" + // size in lines (ignored)
084            "\\S+\\s+" + // size in lines at creation(ignored)
085            "\\S+\\s+" + // lines modified (ignored)
086            "\\S+\\s*"; // id of user who modified (ignored)
087
088    /**
089     * Matches these entries, note: no header:
090     *
091     * <pre>
092     *   IBMUSER1  JOB01906  OUTPUT    3 Spool Files
093     *   012345678901234567890123456789012345678901234
094     *             1         2         3         4
095     * </pre>
096     */
097    static final String JES_LEVEL_1_LIST_REGEX = "(\\S+)\\s+" + // job name ignored
098            "(\\S+)\\s+" + // job number
099            "(\\S+)\\s+" + // job status (OUTPUT,INPUT,ACTIVE)
100            "(\\S+)\\s+" + // number of spool files
101            "(\\S+)\\s+" + // Text "Spool" ignored
102            "(\\S+)\\s*" // Text "Files" ignored
103    ;
104
105    /**
106     * JES INTERFACE LEVEL 2 parser Matches these entries:
107     *
108     * <pre>
109     * JOBNAME  JOBID    OWNER    STATUS CLASS
110     * IBMUSER1 JOB01906 IBMUSER  OUTPUT A        RC=0000 3 spool files
111     * IBMUSER  TSU01830 IBMUSER  OUTPUT TSU      ABEND=522 3 spool files
112     * </pre>
113     *
114     * Sample output from FTP session:
115     *
116     * <pre>
117     * ftp> quote site filetype=jes
118     * 200 SITE command was accepted
119     * ftp> ls
120     * 200 Port request OK.
121     * 125 List started OK for JESJOBNAME=IBMUSER*, JESSTATUS=ALL and JESOWNER=IBMUSER
122     * JOBNAME  JOBID    OWNER    STATUS CLASS
123     * IBMUSER1 JOB01906 IBMUSER  OUTPUT A        RC=0000 3 spool files
124     * IBMUSER  TSU01830 IBMUSER  OUTPUT TSU      ABEND=522 3 spool files
125     * 250 List completed successfully.
126     * ftp> ls job01906
127     * 200 Port request OK.
128     * 125 List started OK for JESJOBNAME=IBMUSER*, JESSTATUS=ALL and JESOWNER=IBMUSER
129     * JOBNAME  JOBID    OWNER    STATUS CLASS
130     * IBMUSER1 JOB01906 IBMUSER  OUTPUT A        RC=0000
131     * --------
132     * ID  STEPNAME PROCSTEP C DDNAME   BYTE-COUNT
133     * 001 JES2              A JESMSGLG       858
134     * 002 JES2              A JESJCL         128
135     * 003 JES2              A JESYSMSG       443
136     * 3 spool files
137     * 250 List completed successfully.
138     * </pre>
139     */
140
141    static final String JES_LEVEL_2_LIST_REGEX = "(\\S+)\\s+" + // job name ignored
142            "(\\S+)\\s+" + // job number
143            "(\\S+)\\s+" + // owner ignored
144            "(\\S+)\\s+" + // job status (OUTPUT,INPUT,ACTIVE) ignored
145            "(\\S+)\\s+" + // job class ignored
146            "(\\S+).*" // rest ignored
147    ;
148
149    private int isType = UNKNOWN_LIST_TYPE;
150
151    /**
152     * Fallback parser for Unix-style listings
153     */
154    private UnixFTPEntryParser unixFTPEntryParser;
155
156    /*
157     * --------------------------------------------------------------------- Very brief and incomplete description of the zOS/MVS-file system. (Note: "zOS" is
158     * the operating system on the mainframe, and is the new name for MVS)
159     *
160     * The file system on the mainframe does not have hierarchal structure as for example the unix file system. For a more comprehensive description, please
161     * refer to the IBM manuals
162     *
163     * @LINK: http://publibfp.boulder.ibm.com/cgi-bin/bookmgr/BOOKS/dgt2d440/CONTENTS
164     *
165     *
166     * Dataset names =============
167     *
168     * A dataset name consist of a number of qualifiers separated by '.', each qualifier can be at most 8 characters, and the total length of a dataset can be
169     * max 44 characters including the dots.
170     *
171     *
172     * Dataset organisation ====================
173     *
174     * A dataset represents a piece of storage allocated on one or more disks. The structure of the storage is described with the field dataset organinsation
175     * (DSORG). There are a number of dataset organisations, but only two are usable for FTP transfer.
176     *
177     * DSORG: PS: sequential, or flat file PO: partitioned dataset PO-E: extended partitioned dataset
178     *
179     * The PS file is just a flat file, as you would find it on the unix file system.
180     *
181     * The PO and PO-E files, can be compared to a single level directory structure. A PO file consist of a number of dataset members, or files if you will. It
182     * is possible to CD into the file, and to retrieve the individual members.
183     *
184     *
185     * Dataset record format =====================
186     *
187     * The physical layout of the dataset is described on the dataset itself. There are a number of record formats (RECFM), but just a few is relavant for the
188     * FTP transfer.
189     *
190     * Any one beginning with either F or V can safely used by FTP transfer. All others should only be used with great care. F means a fixed number of records
191     * per allocated storage, and V means a variable number of records.
192     *
193     *
194     * Other notes ===========
195     *
196     * The file system supports automatically backup and retrieval of datasets. If a file is backed up, the ftp LIST command will return: ARCIVE Not Direct
197     * Access Device KJ.IOP998.ERROR.PL.UNITTEST
198     *
199     *
200     * Implementation notes ====================
201     *
202     * Only datasets that have dsorg PS, PO or PO-E and have recfm beginning with F or V or U, is fully parsed.
203     *
204     * The following fields in FTPFile is used: FTPFile.Rawlisting: Always set. FTPFile.Type: DIRECTORY_TYPE or FILE_TYPE or UNKNOWN FTPFile.Name: name
205     * FTPFile.Timestamp: change time or null
206     *
207     *
208     *
209     * Additional information ======================
210     *
211     * The MVS ftp server supports a number of features via the FTP interface. The features are controlled with the FTP command quote site
212     * filetype=<SEQ|JES|DB2> SEQ is the default and used for normal file transfer JES is used to interact with the Job Entry Subsystem (JES) similar to a job
213     * scheduler DB2 is used to interact with a DB2 subsystem
214     *
215     * This parser supports SEQ and JES.
216     *
217     *
218     *
219     *
220     *
221     *
222     */
223
224    /**
225     * The sole constructor for a MVSFTPEntryParser object.
226     *
227     */
228    public MVSFTPEntryParser() {
229        super(""); // note the regex is set in preParse.
230        super.configure(null); // configure parser with default configurations
231    }
232
233    /*
234     * @return
235     */
236    @Override
237    protected FTPClientConfig getDefaultConfiguration() {
238        return new FTPClientConfig(FTPClientConfig.SYST_MVS, DEFAULT_DATE_FORMAT, null);
239    }
240
241    /**
242     * Parse entries representing a dataset list. Only datasets with DSORG PS or PO or PO-E and with RECFM F[B], V[B], U will be parsed.
243     *
244     * Format of ZOS/MVS file list: 1 2 3 4 5 6 7 8 9 10 Volume Unit Referred Ext Used Recfm Lrecl BlkSz Dsorg Dsname B10142 3390 2006/03/20 2 31 F 80 80 PS
245     * MDI.OKL.WORK ARCIVE Not Direct Access Device KJ.IOP998.ERROR.PL.UNITTEST B1N231 3390 2006/03/20 1 15 VB 256 27998 PO PLU B1N231 3390 2006/03/20 1 15 VB
246     * 256 27998 PO-E PLB
247     *
248     * ----------------------------------- Group within Regex [1] Volume [2] Unit [3] Referred [4] Ext: number of extents [5] Used [6] Recfm: Record format [7]
249     * Lrecl: Logical record length [8] BlkSz: Block size [9] Dsorg: Dataset organisation. Many exists but only support: PS, PO, PO-E [10] Dsname: Dataset name
250     *
251     * Note: When volume is ARCIVE, it means the dataset is stored somewhere in a tape archive. These entries is currently not supported by this parser. A null
252     * value is returned.
253     *
254     * @param entry zosDirectoryEntry
255     * @return null: entry was not parsed.
256     */
257    private FTPFile parseFileList(final String entry) {
258        if (matches(entry)) {
259            final FTPFile file = new FTPFile();
260            file.setRawListing(entry);
261            final String name = group(2);
262            final String dsorg = group(1);
263            file.setName(name);
264
265            // DSORG
266            if ("PS".equals(dsorg)) {
267                file.setType(FTPFile.FILE_TYPE);
268            } else if ("PO".equals(dsorg) || "PO-E".equals(dsorg)) {
269                // regex already ruled out anything other than PO or PO-E
270                file.setType(FTPFile.DIRECTORY_TYPE);
271            } else {
272                return null;
273            }
274
275            return file;
276        }
277
278        return null;
279    }
280
281    /**
282     * Parses a line of an z/OS - MVS FTP server file listing and converts it into a usable format in the form of an <code> FTPFile </code> instance. If the
283     * file listing line doesn't describe a file, then <code> null </code> is returned. Otherwise a <code> FTPFile </code> instance representing the file is
284     * returned.
285     *
286     * @param entry A line of text from the file listing
287     * @return An FTPFile instance corresponding to the supplied entry
288     */
289    @Override
290    public FTPFile parseFTPEntry(final String entry) {
291        switch (isType) {
292        case FILE_LIST_TYPE:
293            return parseFileList(entry);
294        case MEMBER_LIST_TYPE:
295            return parseMemberList(entry);
296        case UNIX_LIST_TYPE:
297            return unixFTPEntryParser.parseFTPEntry(entry);
298        case JES_LEVEL_1_LIST_TYPE:
299            return parseJeslevel1List(entry);
300        case JES_LEVEL_2_LIST_TYPE:
301            return parseJeslevel2List(entry);
302        default:
303            break;
304        }
305
306        return null;
307    }
308
309    /**
310     * Matches these entries, note: no header:
311     *
312     * <pre>
313     * [1]      [2]      [3]   [4] [5]
314     * IBMUSER1 JOB01906 OUTPUT 3 Spool Files
315     * 012345678901234567890123456789012345678901234
316     *           1         2         3         4
317     * -------------------------------------------
318     * Group in regex
319     * [1] Job name
320     * [2] Job number
321     * [3] Job status (INPUT,ACTIVE,OUTPUT)
322     * [4] Number of sysout files
323     * [5] The string "Spool Files"
324     * </pre>
325     *
326     * @param entry zosDirectoryEntry
327     * @return null: entry was not parsed.
328     */
329    private FTPFile parseJeslevel1List(final String entry) {
330        if (matches(entry)) {
331            final FTPFile file = new FTPFile();
332            if (group(3).equalsIgnoreCase("OUTPUT")) {
333                file.setRawListing(entry);
334                final String name = group(2); /* Job Number, used by GET */
335                file.setName(name);
336                file.setType(FTPFile.FILE_TYPE);
337                return file;
338            }
339        }
340
341        return null;
342    }
343
344    /**
345     * Matches these entries:
346     *
347     * <pre>
348     * [1]      [2]      [3]     [4]    [5]
349     * JOBNAME  JOBID    OWNER   STATUS CLASS
350     * IBMUSER1 JOB01906 IBMUSER OUTPUT A       RC=0000 3 spool files
351     * IBMUSER  TSU01830 IBMUSER OUTPUT TSU     ABEND=522 3 spool files
352     * 012345678901234567890123456789012345678901234
353     *           1         2         3         4
354     * -------------------------------------------
355     * Group in regex
356     * [1] Job name
357     * [2] Job number
358     * [3] Owner
359     * [4] Job status (INPUT,ACTIVE,OUTPUT)
360     * [5] Job Class
361     * [6] The rest
362     * </pre>
363     *
364     * @param entry zosDirectoryEntry
365     * @return null: entry was not parsed.
366     */
367    private FTPFile parseJeslevel2List(final String entry) {
368        if (matches(entry)) {
369            final FTPFile file = new FTPFile();
370            if (group(4).equalsIgnoreCase("OUTPUT")) {
371                file.setRawListing(entry);
372                final String name = group(2); /* Job Number, used by GET */
373                file.setName(name);
374                file.setType(FTPFile.FILE_TYPE);
375                return file;
376            }
377        }
378
379        return null;
380    }
381
382    /**
383     * Parse entries within a partitioned dataset.
384     *
385     * Format of a memberlist within a PDS:
386     *
387     * <pre>
388     *    0         1        2          3        4     5     6      7    8
389     *   Name      VV.MM   Created       Changed      Size  Init   Mod   Id
390     *   TBSHELF   01.03 2002/09/12 2002/10/11 09:37    11    11     0 KIL001
391     *   TBTOOL    01.12 2002/09/12 2004/11/26 19:54    51    28     0 KIL001
392     *
393     * -------------------------------------------
394     * [1] Name
395     * [2] VV.MM: Version . modification
396     * [3] Created: yyyy / MM / dd
397     * [4,5] Changed: yyyy / MM / dd HH:mm
398     * [6] Size: number of lines
399     * [7] Init: number of lines when first created
400     * [8] Mod: number of modified lines a last save
401     * [9] Id: User id for last update
402     * </pre>
403     *
404     * @param entry zosDirectoryEntry
405     * @return null: entry was not parsed.
406     */
407    private FTPFile parseMemberList(final String entry) {
408        final FTPFile file = new FTPFile();
409        if (matches(entry)) {
410            file.setRawListing(entry);
411            final String name = group(1);
412            final String datestr = group(2) + " " + group(3);
413            file.setName(name);
414            file.setType(FTPFile.FILE_TYPE);
415            try {
416                file.setTimestamp(super.parseTimestamp(datestr));
417            } catch (final ParseException e) {
418                // just ignore parsing errors.
419                // TODO check this is ok
420                // Drop thru to try simple parser
421            }
422            return file;
423        }
424
425        /*
426         * Assigns the name to the first word of the entry. Only to be used from a safe context, for example from a memberlist, where the regex for some reason
427         * fails. Then just assign the name field of FTPFile.
428         */
429        if (entry != null && !entry.trim().isEmpty()) {
430            file.setRawListing(entry);
431            final String name = entry.split(" ")[0];
432            file.setName(name);
433            file.setType(FTPFile.FILE_TYPE);
434            return file;
435        }
436        return null;
437    }
438
439    /**
440     * preParse is called as part of the interface. Per definition is is called before the parsing takes place. Three kind of lists is recognize: z/OS-MVS File
441     * lists z/OS-MVS Member lists unix file lists
442     *
443     * @since 2.0
444     */
445    @Override
446    public List<String> preParse(final List<String> orig) {
447        // simply remove the header line. Composite logic will take care of the
448        // two different types of
449        // list in short order.
450        if (orig != null && !orig.isEmpty()) {
451            final String header = orig.get(0);
452            if (header.contains("Volume") && header.contains("Dsname")) {
453                setType(FILE_LIST_TYPE);
454                super.setRegex(FILE_LIST_REGEX);
455            } else if (header.contains("Name") && header.contains("Id")) {
456                setType(MEMBER_LIST_TYPE);
457                super.setRegex(MEMBER_LIST_REGEX);
458            } else if (header.indexOf("total") == 0) {
459                setType(UNIX_LIST_TYPE);
460                unixFTPEntryParser = new UnixFTPEntryParser();
461            } else if (header.indexOf("Spool Files") >= 30) {
462                setType(JES_LEVEL_1_LIST_TYPE);
463                super.setRegex(JES_LEVEL_1_LIST_REGEX);
464            } else if (header.indexOf("JOBNAME") == 0 && header.indexOf("JOBID") > 8) {// header contains JOBNAME JOBID OWNER // STATUS CLASS
465                setType(JES_LEVEL_2_LIST_TYPE);
466                super.setRegex(JES_LEVEL_2_LIST_REGEX);
467            } else {
468                setType(UNKNOWN_LIST_TYPE);
469            }
470
471            if (isType != JES_LEVEL_1_LIST_TYPE) { // remove header is necessary
472                orig.remove(0);
473            }
474        }
475
476        return orig;
477    }
478
479    /**
480     * Explicitly set the type of listing being processed.
481     *
482     * @param type The listing type.
483     */
484    void setType(final int type) {
485        isType = type;
486    }
487
488}