· 10 months ago · Nov 22, 2024, 10:30 AM
1#!/usr/bin/env python3.12
2
3import os
4import json
5import logging
6import boto3
7from datetime import datetime, timedelta, timezone
8import sys
9import warnings
10import subprocess
11
12# Suppress deprecation warnings
13warnings.filterwarnings("ignore", category=DeprecationWarning)
14
15# Determine project directory and paths
16SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__))
17PROJECT_DIR = os.path.abspath(os.path.join(SCRIPT_DIR, ".."))
18TMP_DIR = os.path.join(PROJECT_DIR, 'tmp')
19LOG_DIR = os.path.join(PROJECT_DIR, 'var', 'log')
20CONFIG_FILE = os.path.join(PROJECT_DIR, 'etc', 'saadan_s3.json')
21
22# Ensure necessary directories exist
23os.makedirs(TMP_DIR, exist_ok=True)
24os.makedirs(LOG_DIR, exist_ok=True)
25
26# Configure logging
27LOG_FILE = os.path.join(LOG_DIR, 'saadan_s3.log')
28logging.basicConfig(
29 filename=LOG_FILE,
30 level=logging.INFO,
31 format='%(asctime)s - %(levelname)s - %(message)s'
32)
33logging.info("Script started.")
34
35# Load configuration
36try:
37 with open(CONFIG_FILE, 'r') as f:
38 config = json.load(f)
39except FileNotFoundError:
40 logging.error(f"Configuration file {CONFIG_FILE} not found.")
41 sys.exit(1)
42
43# Extract S3 configuration
44endpoint_url = config.get('endpoint_url')
45access_key = config.get('access_key')
46secret_key = config.get('secret_key')
47bucket_name = config.get('bucket_name')
48subdir = config.get('subdir', '') # Use an empty string if not configured
49file_list_filename = config.get('file_list_filename', '') # Default to empty
50deletion_days = config.get('deletion_days', 3) # Default to 3 days if not configured
51post_upload_hook = config.get('post_upload_hook', '') # Command to run after successful upload
52
53# Initialize S3 client
54s3_client = boto3.client(
55 's3',
56 endpoint_url=endpoint_url,
57 aws_access_key_id=access_key,
58 aws_secret_access_key=secret_key,
59)
60
61# Upload progress callback
62def upload_progress(bytes_transferred):
63 print(f"Uploaded {bytes_transferred} bytes", end='\r', flush=True)
64
65def upload_file(file_path):
66 """Upload a file to the specified S3 subdirectory with its original modification time, and make it public."""
67 try:
68 file_name = os.path.basename(file_path)
69 object_key = f"{subdir}/{file_name}" if subdir else file_name
70
71 # Get original modification time
72 mod_time = os.path.getmtime(file_path)
73 mod_time_iso = datetime.utcfromtimestamp(mod_time).isoformat()
74
75 # Upload file with metadata and make it public
76 s3_client.upload_file(
77 file_path,
78 bucket_name,
79 object_key,
80 ExtraArgs={
81 'Metadata': {'OriginalModTime': mod_time_iso},
82 'ACL': 'public-read' # Make the file publicly accessible
83 },
84 Callback=upload_progress
85 )
86 print() # Print a newline after progress
87 logging.info(f"Uploaded {file_name} to {object_key} with modification time {mod_time_iso}, made public")
88
89 # Execute post-upload hook if configured
90 if post_upload_hook:
91 try:
92 hook_command = post_upload_hook.replace('$FILE', file_path)
93 subprocess.run(hook_command, shell=True, check=True)
94 logging.info(f"Executed post-upload hook: {hook_command}")
95 except subprocess.CalledProcessError as e:
96 logging.error(f"Post-upload hook failed: {e}")
97 except Exception as e:
98 logging.error(f"Failed to upload {file_path}: {e}")
99 sys.exit(1)
100
101def delete_old_files():
102 """Delete files in the bucket subdirectory older than the configured number of days."""
103 cutoff_date = datetime.utcnow().replace(tzinfo=timezone.utc) - timedelta(days=deletion_days)
104 try:
105 response = s3_client.list_objects_v2(Bucket=bucket_name, Prefix=subdir)
106 if 'Contents' in response:
107 for obj in response['Contents']:
108 last_modified = obj['LastModified']
109 if last_modified < cutoff_date:
110 s3_client.delete_object(Bucket=bucket_name, Key=obj['Key'])
111 logging.info(f"Deleted {obj['Key']}")
112 except Exception as e:
113 logging.error(f"Failed to delete old files: {e}")
114 sys.exit(1)
115
116def generate_file_list():
117 """Generate a JSON file list and upload it."""
118 if not file_list_filename:
119 logging.info("File list generation is disabled (no file_list_filename configured).")
120 return
121
122 try:
123 file_list = []
124 response = s3_client.list_objects_v2(Bucket=bucket_name, Prefix=subdir)
125
126 if 'Contents' not in response:
127 logging.warning(f"No files found in {bucket_name}/{subdir}")
128 else:
129 for obj in response['Contents']:
130 metadata = s3_client.head_object(Bucket=bucket_name, Key=obj['Key']).get('Metadata', {})
131 file_list.append({
132 'Key': obj['Key'],
133 'LastModified': obj['LastModified'].isoformat(),
134 'Size': obj['Size'],
135 'OriginalModTime': metadata.get('originalmodtime', 'N/A')
136 })
137
138 # Save JSON file locally in TMP_DIR
139 list_file_path = os.path.join(TMP_DIR, file_list_filename)
140 with open(list_file_path, 'w') as f:
141 json.dump(file_list, f, indent=4)
142 logging.info(f"Generated file list at {list_file_path}")
143
144 # Upload JSON file to S3
145 object_key = f"{subdir}/{file_list_filename}" if subdir else file_list_filename
146 s3_client.upload_file(list_file_path, bucket_name, object_key)
147 logging.info(f"Uploaded file list as {object_key}")
148
149 # Remove the local file
150 os.remove(list_file_path)
151 logging.info(f"Deleted local temporary file {list_file_path}")
152 except Exception as e:
153 logging.error(f"Failed to generate or upload file list: {e}")
154 sys.exit(1)
155
156# Main execution
157if __name__ == "__main__":
158 if len(sys.argv) < 2:
159 print("Usage: saadan_s3.py <file_to_upload>")
160 logging.error("No file provided for upload.")
161 sys.exit(1)
162
163 file_to_upload = sys.argv[1]
164 upload_file(file_to_upload) # Upload the file
165 delete_old_files() # Delete old files from the bucket
166 generate_file_list() # Generate and upload file list (if configured)
167 logging.info("Script completed successfully.")
168