kjDQdek8

1#!/usr/bin/python
2#-*- coding: UTF-8 -*-
3
4import re
5import os
6import sys
7import json
8import requests
9import urlparse
10import logging
11import datetime
12
13from tencentcloud.common import credential
14from tencentcloud.common.exception.tencent_cloud_sdk_exception import TencentCloudSDKException
15from tencentcloud.scf.v20180416 import scf_client,models
16from QcloudApi.qcloudapi import QcloudApi
17
18class Job:
19    def __init__(self, config):
20        self.config = config
21
22        # scf api client
23        cred = credential.Credential(config['secret_id'], config['secret_key'])
24        self.scf_client = scf_client.ScfClient(cred, config['scf_region'])
25
26        # cdn api client
27        cdn_config = {
28            'Region':          'ap-guangzhou',
29            'method':          'GET',
30            'secretId':        config['secret_id'],
31            'secretKey':       config['secret_key'],
32            'SignatureMethod': 'HmacSHA1',
33            }
34        self.cdn_client = QcloudApi("cdn", cdn_config)
35        self.cos_path = config['cos_path']
36
37    def get_cdn_log_urls(self, host):
38        '''获取CDN的日志下载链接'''
39        CDN_LOG_STABLE_HOURS = 12+1
40        CDN_LOG_SAVE_HOURS   = 1
41        now = datetime.datetime.now()
42        end = now - datetime.timedelta(hours=CDN_LOG_STABLE_HOURS)
43        start = end - datetime.timedelta(hours=CDN_LOG_SAVE_HOURS)
44
45        action = "GetCdnLogList"
46        action_params = {
47                'host': host,
48                'startDate': start.strftime("%Y-%m-%d %H:%M:%S"),
49                'endDate': end.strftime("%Y-%m-%d %H:%M:%S"),
50                }
51        rsp = self.cdn_client.call(action, action_params)
52        data = json.loads(rsp)
53        if data['code'] != 0:
54            logging.error("API %s error: %s" % (action, data))
55            return []
56        urls = [ v['link'] for v in data['data']['list'] if v['type'] ]
57        if urls:
58            logging.info("time(%s~%s) host[%s] log urls are:\n%s\n." % (start, end, host, "\n".join(urls))  )
59        else:
60            logging.info("time(%s~%s) host[%s] log urls are empty" % (start, end, host) )
61        return urls
62
63    def get_cdn_hosts(self):
64        '''获取账号下全部域名列表'''
65        action = "DescribeCdnHosts"
66        end = datetime.datetime.now()
67        start = end - datetime.timedelta(days=1)
68        action_params = {
69                'detail': 0,
70                }
71        rsp = self.cdn_client.call(action, action_params)
72        data = json.loads(rsp)
73        if data['code'] != 0:
74            logging.error("API %s error: %s" % (action, data))
75            return []
76        hosts = [ v['host'] for v in data['data']['hosts'] ]
77        logging.info("cdn hosts = %s" % hosts)
78        return hosts
79
80    def get_cos_key(self, url):
81        '''
82        解析URL，生成COS上的存储路径格式
83        URL格式为： /day/hour/dayhour-host.gz
84        '''
85        parts = urlparse.urlparse(url)
86        r = r'/(?P<day>[^/]*)/(?P<hour>[^/]*)/(?P<filename>[^-]*-(?P<host>[^/]*).gz)'
87        m = re.match(r, parts.path)
88        if not m:
89            raise RuntimeError("cdn log url format is not support: %s" % url)
90        v = m.groupdict()
91        key = self.cos_path % v
92        return key
93
94    def invoke_cos_upload(self, url):
95        event = dict(self.config)
96        event.update({"url": url, "cos_key": self.get_cos_key(url) })
97        action = "Invoke"
98        action_params = {
99            'InvocationType': "Event", # 异步
100            'FunctionName': self.config['scf_function'],
101            'ClientContext': json.dumps(event),
102        }
103
104        # 调用接口，发起请求，并打印返回结果
105        try:
106            ret = self.scf_client.call(action, action_params)
107            print(json.loads(ret)["Response"]["Result"]["RetMsg"])
108        except TencentCloudSDKException as err:
109            print(err)
110
111
112    def run(self):
113        hosts = self.config['cdn_host']
114        if not hosts:
115            hosts = self.get_cdn_hosts()
116
117        cnt = 0
118        for host in hosts:
119            urls = self.get_cdn_log_urls(host)
120            for url in urls:
121                self.invoke_cos_upload(url)
122            cnt += len(urls)
123        return {"status": "jobs dispatched", "count_url": cnt, "count_host": len(hosts)}
124
125
126def run_app():
127    config = {
128        'secret_id':    '****************',
129        'secret_key':   '***************',
130
131        #COS存储桶的区域
132        'cos_region':   'ap-chengdu',
133        'cos_bucket':   'bucket-1251001234',
134        'cos_path':     '/cdnlog/%(host)s/%(day)s/%(filename)s',
135
136        # SCF配置
137        'scf_region':   'ap-guangzhou',
138        'scf_function': 'cdn-save-log-into-cos',
139
140        # CDN配置
141        # 如果域名列表为空，则表示同步整个账号下全部域名的日志
142        'cdn_host':     [],
143        #'cdn_host':     ['tx-cdn.talebook.org','js.talebook.org'],
144    }
145
146    job = Job(config)
147    return job.run()
148
149def main_handler(event, context):
150    rsp = run_app()
151    return { "statusCode": 200, "body": rsp }