· 8 years ago · May 29, 2018, 03:14 AM
1source $HOME/.profile
2##FILE js_parser.sh
3EU_FILE_LOC=js_parser.sh
4check_args 4 $#
5INFILE=$1;
6OUTPUT=$2;
7FILEORDB=$3;
8DOCROOT=$4;
9
10##C \title{Javascript Parser}
11##C \maketitle
12##C \section{Javascript Parser}
13
14##C \subsection{Introduction}
15##C The purpose of this file is to extract lines of the form
16##C \begin{verbatim}
17##C <script src="../scripts/functions.js" language="javascript"></script>
18##C \end{verbatim}
19##C and then convert them into URLs that can be downloaded.
20##C The output is a 4 column table that looks like
21##C \be
22##C \item lb --- lower bound (inclusive) of character position of script. Non-null
23##C \item ub --- upper bound (exclusive) of character position of script. Non-null
24##C \item name of javascript function. Can be null.
25##C \item inline instructions. Can be null.
26##C \ee
27##C It is possible for both columns 3 and 4 to be null but it is unlikely.
28
29##C \subsection{Algorithm}
30##C \be
31##C \item Delete all files that start with \$prefix. Set to \verb+_jso_+ for now
32prefix=_jso_
33#rm -f ${prefix}*
34##C \item Find all occurrences of text between \verb+<script+ and
35##C \verb+</script>+ and write these to individual files, with prefix.
36## $EUVG extract --from "<script" --to "</script>" --i $INFILE --o $OPFILE\
37## --prefix ${prefix} --lb_ub_file _temp_lb_ub1;
38
39$EUVG rsutils extract "<script" "</script>" $INFILE "" "$prefix" "" "" "" "" _temp_lb_ub1;
40#CHKSTAT
41##C Instruct extractor to print start/stop positions of matches that it finds.
42##C This will serve as columns 1 and 2 of output file.
43# Add header line
44echo "lb,ub" > _temp_lb_ub;
45cat _temp_lb_ub1 >> _temp_lb_ub;
46##C \item Count the number of files produced. If 0, then exit.
47N=`ls -l ${prefix}* | wc -l | sed s'/^[ ]*//'g | cut -f 1 -d ' '`
48is_numeric "$N"
49##C \item For each such file, do the following
50##C \be
51##C \item Extract text from \verb+ src=+ to first space {\bf or}
52##C from \verb+ src="+ to first double quote. This will be column 3.
53##C \item Extract text from \verb+>+ to end of file. This will be column 4.
54##C \ee
55iter=1;
56echo col3 > _temp_col3;
57echo col4 > _temp_col4;
58while [ $iter -le $N ]; do
59 fname=${prefix}$iter;
60 isfile "$fname";
61# $EUVG extract --from " src=\"" --to "\"" --i $fname | \
62 $EUVG rsutils extract " src=\"" "\"" $fname "" "" "" "" "" "" "" | \
63 rsutils replace "" "" NL ";" "" "" | \
64 rsutils file_wspace_to_space | sed s'/"/\\"/'g > _tempf;
65 #CHKSTAT
66 echo "" >> _tempf;
67 cat _tempf | sed s'/^/"/'g | sed s'/$/"/'g > _temp_fld3;
68 sz=`sz_in_blocks _temp_fld3`;
69 if [ $sz = 0 ]; then
70# $EUVG extract --from " src=" --to " " --i $fname | \
71 $EUVG rsutils extract " src=" " " $fname "" "" "" "" "" "" "" | \
72 rsutils replace "" "" NL ";" "" "" | \
73 rsutils file_wspace_to_space | sed s'/"/\\"/'g > _tempf;
74 #CHKSTAT
75 echo "" >> _tempf;
76 cat _tempf | sed s'/^/"/'g | sed s'/$/"/'g > _temp_fld3;
77 sz=`sz_in_blocks _temp_fld3`;
78 if [ $sz = 0 ]; then
79 echo "\"\"" > _temp_fld3;
80 fi
81 fi
82 cat _temp_fld3 >> _temp_col3;
83 #--------------------------------------------------------
84 cat $fname | rsutils replace "" "" NL ";" "" "" > _tempf_1;
85 rsutils rscut ">" "" _tempf_1 > _tempf_2;
86 rsutils keep_m_to_n_chars 1 1000000 "" < _tempf_2 > _tempf_3;
87 rsutils file_wspace_to_space < _tempf_3 | sed s'/"/\\"/'g > _tempf;
88 echo "" >> _tempf;
89 cat _tempf | sed s'/^/"/'g | sed s'/$/"/'g >> _temp_col4;
90 iter=`expr $iter + 1`;
91done
92##C \item Glue column 3 and 4 together
93$EUVG test_glue _js_parser_output _ _ "EOLN" "," _temp_lb_ub _temp_col3 _temp_col4
94#CHKSTAT
95if [ "$FILEORDB" = "FILE" ]; then
96 egrep -v "http:|https:|google-analytics" _js_parser_output > $OUTPUT
97fi
98if [ "$FILEORDB" = "RSDB" ]; then
99 N=`numlines _js_parser_output`;
100 # If there is only a header line, no data
101 if [ $N = 1 ]; then exit 0; fi
102
103 echo "FieldName,Ignore,IFC,Min,Max,CVT2LKP,KEEP" > _js_meta.csv
104 echo "xlb,N,I,,,," >> _js_meta.csv
105 echo "xub,N,I,,,," >> _js_meta.csv
106 echo "fn,N,C,,,," >> _js_meta.csv
107 echo "instr,N,C,,,," >> _js_meta.csv
108 # Set EU_DOC_ROOT if needed; else, make sure it exists
109 if [ "$DOCROOT" != "" ]; then
110 export EU_DOC_ROOT=$DOCROOT;
111 else
112 is_doc_root;
113 fi
114 # Load data
115 x=`rsutils printtimeofday usec`;
116 tempt=tempt_$x
117 $EUVG rsdb "" dld "." _js_parser_output "." _js_meta.csv $tempt;
118 #CHKSTAT
119 # We want to strip away the start <script. Hence the subtraction of 7 */
120 $EUVG rsdb "" add_field $tempt xlb "" 7 "" sub lb "";
121 #CHKSTAT
122 # We want to strip away the end </script>. Hence the addition of 9 */
123 $EUVG rsdb "" add_field $tempt xub "" 9 "" add ub "";
124 #CHKSTAT
125 $EUVG rsdb "" del_flds $tempt "xlb,xub";
126 #CHKSTAT
127 $EUVG rsdb "" match_str $tempt fn "^http" "" "" regex x y;
128 #CHKSTAT
129 N=`$EUVG rsdb "" calc_table_count1 $tempt x ""`;
130 #CHKSTAT
131 M=`$EUVG rsdb "" calc_table_nrec $tempt`;
132 #CHKSTAT
133 if [ $N = 0 ]; then
134 $EUVG rsdb "" rename_tbl $tempt "$OUTPUT";
135 #CHKSTAT
136 else
137 if [ $N -lt $M ]; then
138 $EUVG rsdb "" t_f1_f2_bop $tempt x "" not y;
139 #CHKSTAT
140 $EUVG rsdb "" copy_fields $tempt y $OUTPUT "lb,ub";
141 #CHKSTAT
142 $EUVG rsdb "" xfer_field $tempt fn y $OUTPUT fn
143 #CHKSTAT
144 $EUVG rsdb "" xfer_field $tempt instr y $OUTPUT instr
145 #CHKSTAT
146 fi
147 # Else we do not need to create output table
148 fi
149fi
150##C \ee
151## Cleanup
152rm -f _js_* _temp_lb_ub _temp_col* _tempf_*
153
154##C \end{document}