· 7 years ago · Oct 22, 2018, 12:16 PM
1package main
2
3import (
4 "archive/zip"
5 "encoding/json"
6 "errors"
7 "fmt"
8 "io"
9 "log"
10 "os"
11 "regexp"
12 "strconv"
13 "strings"
14 "time"
15
16 "net/http"
17
18 "github.com/AdRoll/goamz/aws"
19 "github.com/AdRoll/goamz/s3"
20 redigo "github.com/garyburd/redigo/redis"
21)
22
23type Configuration struct {
24 AccessKey string
25 SecretKey string
26 Bucket string
27 Region string
28 RedisServerAndPort string
29 RedisPassword string
30 Port int
31}
32
33var config = Configuration{}
34var aws_bucket *s3.Bucket
35var redisPool *redigo.Pool
36
37type RedisFile struct {
38 FileName string
39 Folder string
40 S3Path string
41 // Optional - we use are Teamwork.com but feel free to rmove
42 FileId int64 `json:",string"`
43 ProjectId int64 `json:",string"`
44 ProjectName string
45 Modified string
46 ModifiedTime time.Time
47}
48
49func main() {
50 if 1 == 0 {
51 test()
52 return
53 }
54
55 configFile, _ := os.Open("conf.json")
56 decoder := json.NewDecoder(configFile)
57 err := decoder.Decode(&config)
58 if err != nil {
59 panic("Error reading conf")
60 }
61
62 fmt.Println("config file", config)
63
64 initAwsBucket()
65 InitRedis()
66
67 fmt.Println("Running on port", config.Port)
68 http.HandleFunc("/", handler)
69 http.ListenAndServe(":"+strconv.Itoa(config.Port), nil)
70}
71
72func test() {
73 var err error
74 var files []*RedisFile
75 jsonData := "[{\"S3Path\":\"1\\/p23216.tf_A89A5199-F04D-A2DE-5824E635AC398956.Avis_Rent_A_Car_Print_Reservation.pdf\",\"FileVersionId\":\"4164\",\"FileName\":\"Avis Rent A Car_ Print Reservation.pdf\",\"ProjectName\":\"Superman\",\"ProjectId\":\"23216\",\"Folder\":\"\",\"FileId\":\"4169\"},{\"modified\":\"2015-07-18T02:05:04Z\",\"S3Path\":\"1\\/p23216.tf_351310E0-DF49-701F-60601109C2792187.a1.jpg\",\"FileVersionId\":\"4165\",\"FileName\":\"a1.jpg\",\"ProjectName\":\"Superman\",\"ProjectId\":\"23216\",\"Folder\":\"Level 1\\/Level 2 x\\/Level 3\",\"FileId\":\"4170\"}]"
76
77 resultByte := []byte(jsonData)
78
79 err = json.Unmarshal(resultByte, &files)
80 if err != nil {
81 err = errors.New("Error decoding json: " + jsonData)
82 }
83}
84
85func initAwsBucket() {
86 expiration := time.Now().Add(time.Hour * 1)
87 auth, err := aws.GetAuth(config.AccessKey, config.SecretKey, "", expiration) //"" = token which isn't needed
88 if err != nil {
89 panic(err)
90 }
91
92 aws_bucket = s3.New(auth, aws.GetRegion(config.Region)).Bucket(config.Bucket)
93}
94
95func InitRedis() {
96 redisPool = &redigo.Pool{
97 MaxIdle: 10,
98 IdleTimeout: 1 * time.Second,
99 Dial: func() (redigo.Conn, error) {
100 c, err := redigo.Dial("tcp", config.RedisServerAndPort)
101 if err != nil {
102 return nil, err
103 }
104 if _, err := c.Do("AUTH", config.RedisPassword); err != nil {
105 c.Close()
106 return nil, err
107 }
108 if _, err := c.Do("SELECT", 0); err != nil {
109 c.Close()
110 return nil, err
111 }
112 return c, nil
113 },
114 TestOnBorrow: func(c redigo.Conn, t time.Time) (err error) {
115 _, err = c.Do("PING")
116 if err != nil {
117 panic("Error connecting to redis")
118 }
119 return
120 },
121 }
122}
123
124// Remove all other unrecognised characters apart from
125var makeSafeFileName = regexp.MustCompile(`[#<>:"/\|?*\\]`)
126
127func getFilesFromRedis(ref string) (files []*RedisFile, err error) {
128
129 // Testing - enable to test. Remove later.
130 if 1 == 0 && ref == "test" {
131 files = append(files, &RedisFile{FileName: "test.zip", Folder: "", S3Path: "test/test.zip"}) // Edit and dplicate line to test
132 return
133 }
134
135fmt.Println("redisPool", redisPool)
136
137 redis := redisPool.Get()
138 defer redis.Close()
139
140 // Get the value from Redis
141 result, err := redis.Do("GET", "zip:"+ref)
142 if err != nil || result == nil {
143 err = errors.New("Access Denied (sorry your link has timed out)")
144 return
145 }
146 redis.Do("DEL", "zip:"+ref)
147 // Convert to bytes
148 var resultByte []byte
149 var ok bool
150 if resultByte, ok = result.([]byte); !ok {
151 err = errors.New("Error converting data stream to bytes")
152 return
153 }
154
155 // Decode JSON
156 err = json.Unmarshal(resultByte, &files)
157 if err != nil {
158 err = errors.New("Error decoding json: " + string(resultByte))
159 }
160
161 return
162}
163
164func handler(w http.ResponseWriter, r *http.Request) {
165 start := time.Now()
166
167 health, ok := r.URL.Query()["health"]
168 if len(health) > 0 {
169 fmt.Fprintf(w, "OK")
170 return
171 }
172
173 // Get "ref" URL params
174 refs, ok := r.URL.Query()["ref"]
175 if !ok || len(refs) < 1 {
176 http.Error(w, "S3 File Zipper. Pass ?ref= to use.", 500)
177 return
178 }
179 ref := refs[0]
180
181 // Get "downloadas" URL params
182 downloadas, ok := r.URL.Query()["downloadas"]
183 if !ok && len(downloadas) > 0 {
184 downloadas[0] = makeSafeFileName.ReplaceAllString(downloadas[0], "")
185 if downloadas[0] == "" {
186 downloadas[0] = "download.zip"
187 }
188 } else {
189 downloadas = append(downloadas, "download.zip")
190 }
191
192 files, err := getFilesFromRedis(ref)
193 if err != nil {
194 http.Error(w, err.Error(), 403)
195 log.Printf("%s\t%s\t%s", r.Method, r.RequestURI, err.Error())
196 return
197 }
198
199 // Start processing the response
200 w.Header().Add("Content-Disposition", "attachment; filename=\""+downloadas[0]+"\"")
201 w.Header().Add("Content-Type", "application/zip")
202
203 // Loop over files, add them to the
204 zipWriter := zip.NewWriter(w)
205 t := time.Now()
206 for _, file := range files {
207
208 if file.S3Path == "" {
209 log.Printf("Missing path for file: %v", file)
210 continue
211 }
212
213 // Build safe file file name
214 safeFileName := makeSafeFileName.ReplaceAllString(file.FileName, "")
215 if safeFileName == "" { // Unlikely but just in case
216 safeFileName = "file"
217 }
218
219 // Read file from S3, log any errors
220 rdr, err := aws_bucket.GetReader(file.S3Path)
221 if err != nil {
222 switch t := err.(type) {
223 case *s3.Error:
224 if t.StatusCode == 404 {
225 log.Printf("File not found. %s", file.S3Path)
226 }
227 default:
228 log.Printf("Error downloading \"%s\" - %s", file.S3Path, err.Error())
229 }
230 continue
231 }
232
233 // Build a good path for the file within the zip
234 zipPath := ""
235 // Prefix project Id and name, if any (remove if you don't need)
236 if file.ProjectId > 0 {
237 zipPath += strconv.FormatInt(file.ProjectId, 10) + "."
238 // Build Safe Project Name
239 file.ProjectName = makeSafeFileName.ReplaceAllString(file.ProjectName, "")
240 if file.ProjectName == "" { // Unlikely but just in case
241 file.ProjectName = "Project"
242 }
243 zipPath += file.ProjectName + "/"
244 }
245 // Prefix folder name, if any
246 if file.Folder != "" {
247 zipPath += file.Folder
248 if !strings.HasSuffix(zipPath, "/") {
249 zipPath += "/"
250 }
251 }
252 zipPath += safeFileName
253
254 // We have to set a special flag so zip files recognize utf file names
255 // See http://stackoverflow.com/questions/30026083/creating-a-zip-archive-with-unicode-filenames-using-gos-archive-zip
256 h := &zip.FileHeader{
257 Name: zipPath,
258 Method: zip.Deflate,
259 Flags: 0x800,
260 }
261
262 h.SetModTime(t)
263
264 f, _ := zipWriter.CreateHeader(h)
265
266 io.Copy(f, rdr)
267 rdr.Close()
268 }
269
270 zipWriter.Close()
271
272 log.Printf("%s\t%s\t%s", r.Method, r.RequestURI, time.Since(start))
273}