· 4 years ago · May 08, 2021, 07:58 AM
1package com.example.backend.scheduler;
2
3import java.io.IOException;
4import java.io.InputStream;
5import java.net.HttpURLConnection;
6import java.net.URI;
7import java.net.URL;
8import java.net.URLEncoder;
9import java.net.http.HttpClient;
10import java.net.http.HttpRequest;
11import java.net.http.HttpResponse;
12import java.nio.charset.StandardCharsets;
13import java.nio.file.Files;
14import java.nio.file.Path;
15import java.nio.file.Paths;
16import java.time.Duration;
17import java.util.*;
18import java.util.stream.Collectors;
19
20import static java.nio.file.Files.writeString;
21
22public class GetJsonGetSqlApi {
23
24 //SQLizer API key
25 private static String API_KEY = "4ejggYADZS1VtkAgNQnz0TZp8r8AKjq3LowSUqjBIERJMvo3khCkc9DqXzmSWDnB1Co6WwtctjxR7bgxgkzUNA";
26 // URL to download JSON from
27 private static String jsonUrl = "https://cosmos-odyssey.azurewebsites.net/api/v1.0/TravelPrices";
28 // where to save downloaded SQL
29 private static String saveDir = "backend/src/main/resources/downloads";
30
31 public static void main(String[] args) throws IOException, InterruptedException {
32
33 String json = downloadJson(jsonUrl);
34
35
36 HttpClient client = HttpClient.newHttpClient();
37 //GET FILE ID
38
39 Map<String, String> parameters = new HashMap<>();
40 parameters.put("FileType", "json");
41 parameters.put("FileName", "file.json");
42 parameters.put("TableName", "flights");
43 parameters.put("DatabaseType", "PostgreSQL");
44 parameters.put("FileHasHeaders", "false");
45
46 String form = parameters.keySet().stream()
47 .map(key -> key + "=" + URLEncoder.encode(parameters.get(key), StandardCharsets.UTF_8))
48 .collect(Collectors.joining("&"));
49
50 HttpRequest postUploadFileInitialize = HttpRequest.newBuilder()
51 .uri(URI.create("https://sqlizer.io/api/files/"))
52 .setHeader("Authorization", "Bearer " + API_KEY)
53 .setHeader("Content-Type", "application/x-www-form-urlencoded")
54 .POST(HttpRequest.BodyPublishers.ofString(form))
55 .build();
56
57 HttpResponse<String> response = client.send(postUploadFileInitialize, HttpResponse.BodyHandlers.ofString());
58 String bracesRemoved = response.body().replace("{", "").replace("}", "");
59
60 String id = "";
61 for (String possible : bracesRemoved.split(",")) {
62 if (possible.contains("ID")) {
63 String quoted = possible.split(":")[1];
64 id = quoted.replaceAll("\"", "");
65 }
66 }
67
68
69 // UPLOAD
70
71 List<String> file = new ArrayList<>(Files.readAllLines(Path.of("backend/src/main/resources/downloads/file.json")));
72 List<byte[]> bytes = new ArrayList<>();
73 bytes.add("--ThisIsABoundryWeeeeeeee\r\nContent-Disposition: form-data; name=\"file\"; filename=\"file.json\"\r\nContent-Type: application/octet-stream\r\n\r\n".getBytes(StandardCharsets.UTF_8));
74 file.forEach(line -> bytes.add((line + "\r\n").getBytes(StandardCharsets.UTF_8)));
75 bytes.add("--ThisIsABoundryWeeeeeeee--".getBytes(StandardCharsets.UTF_8));
76
77 byte[] result = flatByteList(bytes);
78 HttpRequest postUploadFile = HttpRequest.newBuilder()
79 .timeout(Duration.ofSeconds(60))
80 .uri(URI.create("https://sqlizer.io/api/files/" + id + "/data/"))
81 .setHeader("Authorization", "Bearer " + API_KEY)
82 .setHeader("Content-Type", "multipart/form-data; boundary=ThisIsABoundryWeeeeeeee")
83 .POST(HttpRequest.BodyPublishers.ofByteArray(result))
84 .build();
85 HttpResponse<String> uploadResponse = client.send(postUploadFile, HttpResponse.BodyHandlers.ofString());
86 System.out.println(uploadResponse.body());
87 Thread.sleep(5000);
88
89 HttpRequest getResult = HttpRequest.newBuilder()
90 .uri(URI.create("https://sqlizer.io/api/files/" + id))
91 .setHeader("Authorization", "Bearer " + API_KEY)
92 .setHeader("Content-Type", "application/x-www-form-urlencoded")
93 .GET()
94 .build();
95
96 HttpResponse<String> resultssss = client.send(getResult, HttpResponse.BodyHandlers.ofString());
97 System.out.println(resultssss.body());
98 Thread.sleep(2000);
99
100 // PUT
101
102 Map<String, String> putParameters = new HashMap<>();
103 putParameters.put("Status", "Uploaded");
104
105 String putForm = putParameters.keySet().stream()
106 .map(key -> key + "=" + URLEncoder.encode(putParameters.get(key), StandardCharsets.UTF_8))
107 .collect(Collectors.joining("&"));
108
109 HttpRequest putUploadConvert = HttpRequest.newBuilder()
110 .timeout(Duration.ofSeconds(60))
111 .uri(URI.create("https://sqlizer.io/api/files/" + id))
112 .setHeader("Authorization", "Bearer " + API_KEY)
113 .setHeader("Content-Type", "application/x-www-form-urlencoded")
114 .PUT(HttpRequest.BodyPublishers.ofString(putForm))
115 .build();
116
117 HttpResponse<String> putResponse = client.send(putUploadConvert, HttpResponse.BodyHandlers.ofString());
118 System.out.println(putResponse.body());
119
120
121 //CHECK STATUS
122 String status = "";
123 String getLink = "";
124
125 while (!status.equals("Complete")) {
126
127 HttpRequest checkStatus = HttpRequest.newBuilder()
128 .timeout(Duration.ofSeconds(60))
129 .uri(URI.create("https://sqlizer.io/api/files/" + id))
130 .setHeader("Authorization", "Bearer " + API_KEY)
131 .setHeader("Content-Type", "application/x-www-form-urlencoded")
132 .GET()
133 .build();
134
135 HttpResponse<String> statusResponse = client.send(checkStatus, HttpResponse.BodyHandlers.ofString());
136 System.out.println(statusResponse.body());
137 Thread.sleep(500);
138 String statusRemoveBraces = statusResponse.body().replace("{", "").replace("}", "");
139 String resultUrlRemoveBraces = statusResponse.body().replace("{", "").replace("}", "");
140
141 for (String possible : statusRemoveBraces.split(",")) {
142 if (possible.contains("Status")) {
143 String quoted = possible.split(":")[1];
144 status = quoted.replaceAll("\"", "");
145 }
146 }
147
148 // extract url, make browser friendly
149 for (String possible : resultUrlRemoveBraces.split(",")) {
150 if (possible.contains("ResultUrl")) {
151 String quoted = possible.split(":")[2];
152 getLink = "https:" + quoted.replaceAll("\"", "").replace("\\u0026", "&");
153 }
154 }
155 }
156
157 System.out.println("{\"Status\":\"Get Link\"} " + getLink);
158
159 try {
160 HttpDownloadUtility.downloadFile(getLink, saveDir);
161 } catch (IOException ex) {
162 ex.printStackTrace();
163 }
164 }
165
166 private static String downloadJson(String jsonUrl) {
167 //DOWNLOAD JSON
168 String json = null;
169 try {
170 URL url = new URL(jsonUrl);
171 HttpURLConnection connection = (HttpURLConnection) url.openConnection();
172 connection.setDoOutput(true);
173 connection.setInstanceFollowRedirects(false);
174 connection.setRequestMethod("GET");
175 connection.setRequestProperty("Content-Type", "application/json");
176 connection.setRequestProperty("charset", "utf-8");
177 connection.connect();
178 InputStream inStream = connection.getInputStream();
179 json = streamToString(inStream); // input stream to string
180
181 // save string as JSON
182 Path path = Paths.get("backend/src/main/resources/downloads/file.json");
183 String contents = json;
184 try {
185 writeString(path, contents, StandardCharsets.UTF_8);
186 } catch (IOException ex) {
187 // TODO Handle exception
188 }
189
190 } catch (IOException ex) {
191 ex.printStackTrace();
192 }
193 System.out.println("{\"Status\":\"JSON Downloaded\"} " + json.substring(1, 60) + "...");
194 return json;
195 }
196
197
198 private static String streamToString(InputStream inputStream) {
199 String text = new Scanner(inputStream, "UTF-8").useDelimiter("\\Z").next();
200 return text;
201 }
202
203 private static byte[] flatByteList(List<byte[]> list) {
204 int length = 0;
205 for (byte[] byteArr : list) {
206 length += byteArr.length;
207 }
208 byte[] result = new byte[length];
209 int pos = 0;
210 for (byte[] arr : list) {
211 for (byte b : arr) {
212 result[pos++] = b;
213 }
214 }
215 return result;
216 }
217
218}
219