b9CU2NDJ

1#!/usr/bin/env bash
2
3# @FeedPhish bulk URL Uploader script.
4# Want a key? Message us on twitter: https://twitter.com/FeedPhish
5# Usage:
6# ./PhishFeed.sh --submit urls.txt
7
8
9
10set -Eeuo pipefail
11
12endpoint="https://api.phishfeed.com/submissions/v1/submit-url"
13MAX_URLS_PER_REQUEST=10000
14
15declare -A client_keys
16
17# Insert your API key
18client_keys["main"]="API_KEY"
19
20
21need-jq() {
22    echo "'jq' is required."
23    echo
24    echo "If you are running Ubuntu, you can install it with:"
25    echo "sudo apt-get install jq"
26    exit 1
27}
28
29usage() {
30    echo "Usage: ${0} <CLIENT> [--submit] [FILE]..."
31    echo
32    echo "  --submit    Post data to the API."
33    echo "  --help      Show this message."
34    echo
35    echo "Default behavior is to dry-run and show what would be sent."
36    echo
37    echo "FILE          A file with URL data to post"
38    echo
39}
40
41declare -A flags
42declare -a incoming_files
43
44jq_file() {
45    infile="$1"
46    outfile="$2"
47    jq -R "." < "${infile}" | jq -s '{"urls": .}' > "${outfile}"
48}
49
50submit_file() {
51    datafile="$1"
52    if [ "${flags['SUBMIT']:-0}" -eq 1 ]; then
53        curl -X POST \
54             -H "x-api-key: ${client_keys[${CLIENTNAME}]}" \
55             "${endpoint}" \
56             --data-binary @"${datafile}"
57    else
58    	usage
59    fi
60}
61
62process_file() {
63    file="$1"
64    jq_file "${file}" "${file}.json"
65    submit_file "${file}.json"
66}
67
68if [ $# -eq 0 ]; then
69    echo "No arguments provided."
70    usage
71    exit 1
72fi
73
74CLIENTNAME="main"
75if [ "${1}" == "--help" ]; then
76    usage
77    exit 0
78fi
79
80for arg in "$@"; do
81    case $arg in
82        --submit)
83            flags["SUBMIT"]=1
84            ;;
85        --help)
86            usage
87            exit 0
88            ;;
89        -*)
90            echo "Unsupported argument '${arg}'."
91            flags["ABORT"]=1
92            ;;
93        *)
94            if [ -f "${arg}" ]; then
95                incoming_files+=("${arg}")
96            else
97                echo "Not a valid file: '${arg}'"
98            fi
99            ;;
100    esac
101done
102
103if [ "${flags['ABORT']:-0}" -eq 1 ]; then
104    echo "Invalid arguments provided, aborting."
105    exit 1
106fi
107
108if [ -z "${incoming_files[*]}" ]; then
109    echo "No files provided to process, aborting."
110    exit 1
111fi
112
113cleanup_on_exit() {
114    for file in "${incoming_files[@]}"; do
115        if [ "${flags['DEBUG']:-0}" -eq 1 ]; then
116            echo "Cleaning up for '${file}'"
117            ls -l "${file}"{.json,.split*} 2>/dev/null || true
118        fi
119        rm -f "${file}"{.json,.split*}
120    done
121}
122
123trap 'cleanup_on_exit' EXIT
124for file in "${incoming_files[@]}"; do
125    if [ "$(wc -l < "${file}")" -gt "${MAX_URLS_PER_REQUEST}" ]; then
126        echo "big file"
127        split \
128            --suffix-length 3 \
129            --numeric-suffixes=1 \
130            --lines="${MAX_URLS_PER_REQUEST}" \
131            "${file}" \
132            "${file}.split"
133        for smaller_file in "${file}.split"*; do
134            process_file "${smaller_file}"
135        done
136    else
137        process_file "${file}"
138    fi
139done
140
141