· 8 years ago · Feb 02, 2018, 03:48 PM
1#!/usr/bin/env python
2
3import struct
4from MythTV import MythDB, MythBE, Recorded, MythError, Video, VideoGrabber
5import pytz
6import shlex, subprocess
7import os, sys, signal, shutil
8import tempfile
9import time
10import urllib
11import pprint
12
13# Calculate elapsed time and return it as a pretty string.
14# "start" must be a time.time() value.
15def elapsedTime(start):
16 end = time.time()
17 hours, rem = divmod(end-start, 3600)
18 minutes, seconds = divmod(rem, 60)
19 outstr = "{} seconds".format(int(round(seconds)))
20 if minutes > 0:
21 outstr = "{} minutes, {}".format(int(minutes), outstr)
22 if hours > 0:
23 outstr = "{} hours, {}".format(int(hours), outstr)
24 return outstr
25
26# Opensubtitles style file hash stolen from
27# https://trac.opensubtitles.org/projects/opensubtitles/wiki/HashSourceCodes
28def hashFile(name):
29 try:
30
31 longlongformat = '<q' # little-endian long long
32 bytesize = struct.calcsize(longlongformat)
33
34 f = open(name, "rb")
35
36 filesize = os.path.getsize(name)
37 hash = filesize
38
39 if filesize < 65536 * 2:
40 return "SizeError"
41
42 for x in range(65536/bytesize):
43 buffer = f.read(bytesize)
44 (l_value,)= struct.unpack(longlongformat, buffer)
45 hash += l_value
46 hash = hash & 0xFFFFFFFFFFFFFFFF #to remain as 64bit number
47
48
49 f.seek(max(0,filesize-65536),0)
50 for x in range(65536/bytesize):
51 buffer = f.read(bytesize)
52 (l_value,)= struct.unpack(longlongformat, buffer)
53 hash += l_value
54 hash = hash & 0xFFFFFFFFFFFFFFFF
55
56 f.close()
57 returnedhash = "%016x" % hash
58 return returnedhash
59
60 except(IOError):
61 return ''
62
63# Download coverart and screenshot images named in the metadata
64def getImage(filename, ftype, url, hostname):
65 try:
66 target = ''
67 if (ftype == 'coverart'):
68 for sg in DB.getStorageGroup(groupname="Coverart", hostname=hostname):
69 target = sg.dirname
70 elif (ftype == 'screenshot'):
71 for sg in DB.getStorageGroup(groupname="Screenshots", hostname=hostname):
72 target = sg.dirname
73 elif (ftype == 'banner'):
74 for sg in DB.getStorageGroup(groupname="Banners", hostname=hostname):
75 target = sg.dirname
76
77 if (target != ''):
78 # Set the target file name
79 target = os.path.join(target, filename)
80 # Download the file if we don't already have it
81 if (not os.path.isfile(target)):
82 print " Downloading {} image file: {}".format(ftype, os.path.basename(target))
83 image = urllib.URLopener()
84 image.retrieve(url, target)
85 else:
86 print " Using existing {} image file: {}".format(ftype, os.path.basename(target))
87 return target
88 except Exception as e:
89 _type, _obj, _tb = sys.exc_info()
90 print "Unexpected error at line {} in getImage function".format(_tb.tb_lineno)
91 print "{}: {}".format(e.errno, e.strerror)
92 return ''
93
94elapsed_start = time.time()
95
96print "Getting list of recordings ..."
97
98# Get a list of all of the recordings
99class MyRecorded(Recorded):
100 _table = 'recorded'
101
102DB = MythDB()
103DB.searchRecorded.handler = MyRecorded
104DB.searchRecorded.dbclass = MyRecorded
105
106recordings = list(DB.searchRecorded())
107
108# Ask the user to select a program from the list of recordings
109title_list = []
110counter = 1
111print "\nPlease select the program name:"
112for recording in sorted(recordings, key=lambda x: x.title):
113 if not recording.title in title_list:
114 print "{}: {}".format(counter, recording.title)
115 title_list.append(recording.title)
116 counter += 1
117recording_id = int(raw_input("Enter number: "))
118episode_list = []
119counter = 1
120print "\nPlease select the episode to be converted:"
121for recording in sorted(recordings, key=lambda x: x.starttime):
122 if recording.title == title_list[recording_id - 1]:
123 episode_list.append(recording)
124 episode_title = recording.title
125 if recording.subtitle != '':
126 episode_title = "{} - {}".format(episode_title, recording.subtitle)
127 print "{}: {} - {:%Y-%m-%d %I:%M:%S %p}".format(counter, episode_title, recording.starttime)
128 counter += 1
129episode_id = int(raw_input("Enter number: "))
130
131# This is the recording that we'll convert
132recording = episode_list[episode_id - 1]
133
134print "\nGetting recording details from local sources ...\n"
135
136# Get the path to the recording's storage group
137for sg in DB.getStorageGroup(groupname=recording.storagegroup, hostname=recording.hostname):
138 storage_path = sg.dirname
139
140# Calculate additional recording metadata items
141recording.year = recording.starttime.strftime("%Y")
142delta = recording.endtime - recording.starttime
143recording.duration = int(delta.total_seconds() / 60)
144
145# Get video characteristics using mythtranscode utility
146cmd = shlex.split('mythtranscode -v general --passthrough --fifoinfo --chanid {} --starttime {}'.format(recording.chanid, recording.starttime.astimezone(pytz.utc).strftime('%Y%m%d%H%M%S')))
147proc = subprocess.Popen(cmd, stdout=subprocess.PIPE)
148while True:
149 line = proc.stdout.readline()
150 if line != '':
151 if 'FifoVideoWidth' in line:
152 video_width = line.split()[-1]
153 if 'FifoVideoHeight' in line:
154 video_height = line.split()[-1]
155 if 'FifoVideoAspectRatio' in line:
156 video_aspect = line.split()[-1]
157 if 'FifoVideoFrameRate' in line:
158 video_fps = line.split()[-1]
159 if 'FifoAudioFormat' in line:
160 audio_format = line.split()[-1]
161 if 'FifoAudioChannels' in line:
162 audio_channels = line.split()[-1]
163 if 'FifoAudioSampleRate' in line:
164 audio_rate = line.split()[-1]
165 else:
166 break
167
168# Deal with tweaks and special cases
169if recording.title == "Christopher Kimball's Milk Street Television":
170 recording.title = "Milk Street Television"
171 recording.inetref = "334532"
172if recording.title == "America's Test Kitchen From Cook's Illustrated":
173 recording.title = "America's Test Kitchen"
174elif recording.title == "Cook's Country from America's Test Kitchen" or recording.title == "Cook's Country From America's Test Kitchen":
175 recording.title = "Cook's Country"
176elif recording.title == "Frontline":
177 recording.inetref = "99999999"
178 recording.season = recording.year
179 recording.episode = 1
180 for v in DB.searchVideos(title="Frontline", year=recording.season):
181 recording.episode += 1
182elif recording.title == 'Kathy':
183 recording.inetref = "257842"
184
185# Try to get video metadata from TheTVDB web site
186meta = {}
187grab = VideoGrabber('TV')
188if recording.season == 0 and recording.subtitle != '':
189 print "Looking up recording at TheTVDB.com using title and subtitle"
190 try:
191 results = grab.sortedSearch(recording.title, subtitle=recording.subtitle)
192 if len(results) > 0:
193 for show in results:
194 if show.title == recording.title:
195 recording.season = show.season
196 recording.episode = show.episode
197 meta = show
198 except:
199 pass
200elif recording.season != 0:
201 print "Looking up recording at TheTVDB.com using inetref, season and episode numbers"
202 if recording.inetref == '':
203 try:
204 results = grab.sortedSearch(recording.title, tolerance=10)
205 if len(results) > 0:
206 for show in results:
207 if show.title == recording.title:
208 recording.inetref = show.interef
209 except:
210 pass
211 else:
212 if '_' in recording.inetref:
213 recording.inetref = recording.inetref.split('_')[1]
214 if recording.inetref != '':
215 try:
216 meta = grab.grabInetref(recording.inetref, season=recording.season, episode=recording.episode)
217 except:
218 pass
219if meta:
220 print "Got metadata from TheTVDB.com"
221
222# Build output file name
223for sg in DB.getStorageGroup(groupname="Videos", hostname=recording.hostname):
224 videos_path = sg.dirname
225target_path = os.path.join(videos_path, "TV", recording.title, "Season {:02d}".format(int(recording.season)))
226if recording.subtitle != "":
227 target_file = os.path.join(target_path, "{}.S{:02d}E{:02d}.{}.HDTV.x264.mp4".format(recording.title, int(recording.season), int(recording.episode), recording.subtitle).replace(" ", "."))
228else:
229 target_file = os.path.join(target_path, "{}.S{:02d}E{:02d}.HDTV.x264.mp4".format(recording.title, int(recording.season), int(recording.episode)).replace(" ", "."))
230
231# Check to see if this recording already exists
232if os.path.exists(target_file):
233 raise Exception('{} already exists'.format(target_file))
234
235print "\nRecording details:"
236print " Title : {}".format(recording.title)
237print " Subtitle : {}".format(recording.subtitle)
238print " Season : {}".format(recording.season)
239print " Episode : {}".format(recording.episode)
240print " Plot : {}".format(recording.description)
241print " Category : {}".format(recording.category)
242print " Year : {}".format(recording.year)
243print " Program date/time : {:%Y-%m-%d %I:%M:%S %p}".format(recording.starttime)
244print " Duration : {}".format(recording.duration)
245print " Channel : {}".format(recording.chanid)
246print " StartTime : {:%Y%m%d%H%M%S}".format(recording.starttime.astimezone(pytz.utc))
247print " Recording file : {}".format(os.path.join(storage_path, recording.basename))
248print " Audio format : {}".format(audio_format)
249print " Audio rate : {}".format(audio_rate)
250print " Audio channels : {}".format(audio_channels)
251print " Video size : {}x{}".format(video_width, video_height)
252print " Video frame rate : {}".format(video_fps)
253print " Video aspect ratio: {}".format(video_aspect)
254print " Target file : {}".format(target_file)
255
256# Prompt for options
257answ = raw_input("\nModify season and/or episode numbers [y|N]: ")
258if answ.upper() == 'Y':
259 num = ''
260 while num == '':
261 num = raw_input("Enter season number [{}]: ".format(recording.season)).strip()
262 if num == '':
263 num = recording.season
264 try:
265 num = int(num)
266 except:
267 num = ''
268 recording.season = num
269 if meta:
270 meta['season'] = num
271
272 num = ''
273 while num == '':
274 num = raw_input("Enter episode number [{}]: ".format(recording.episode)).strip()
275 if num == '':
276 num = recording.episode
277 try:
278 num = int(num)
279 except:
280 num = ''
281 recording.episode = num
282 if meta:
283 meta['episode'] = num
284
285 # Reconstruct output file name
286 target_path = os.path.join(videos_path, "TV", recording.title, "Season {:02d}".format(int(recording.season)))
287 if recording.subtitle != "":
288 target_file = os.path.join(target_path, "{}.S{:02d}E{:02d}.{}.HDTV.x264.mp4".format(recording.title, int(recording.season), int(recording.episode), recording.subtitle).replace(" ", "."))
289 else:
290 target_file = os.path.join(target_path, "{}.S{:02d}E{:02d}.HDTV.x264.mp4".format(recording.title, int(recording.season), int(recording.episode)).replace(" ", "."))
291 print "New target file: {}".format(target_file)
292
293 # Check to see if this recording already exists
294 if os.path.exists(target_file):
295 raise Exception('{} already exists'.format(target_file))
296
297cut = False
298answ = raw_input("Generate cut list from commercial flags? [y|N]: ")
299if answ.upper() == 'Y':
300 cut = True
301
302normalize = True
303answ = raw_input("Normalize the audio? [Y|n]: ")
304if answ.upper() == 'N':
305 normalize = False
306
307delogo = ''
308answ = raw_input("Crop a WGVU-Life program? [y|N]: ")
309if answ.upper() == 'Y':
310 delogo = "-vf crop=704:356:0:60"
311 video_aspect = "1.77778"
312else:
313 answ = raw_input("Crop and delogo a WGVU Create-TV program? [y|N]: ")
314 if answ.upper() == 'Y':
315 delogo = "-vf delogo=x=55:y=400:w=104:h=25:band=5,crop=704:356:0:60"
316 video_aspect = "1.77778"
317 else:
318 answ = raw_input("Remove the new WGVU logo? [y|N]: ")
319 if answ.upper() == 'Y':
320 if video_height == '720':
321 delogo = "-vf delogo=x=905:y=636:w=138:h=31:band=5"
322 else:
323 if video_width == '1440':
324 delogo = "-vf delogo=x=1220:y=985:w=149:h=40:band=10"
325 else:
326 delogo = "-vf delogo=x=1626:y=985:w=199:h=40:band=10"
327 else:
328 answ = raw_input("Remove the old WGVU logo? [y|N]: ")
329 if answ.upper() == 'Y':
330 if video_height == '720':
331 delogo = "-vf delogo=x=905:y=636:w=138:h=31:band=5"
332 else:
333 delogo = "-vf delogo=x=1359:y=950:w=212:h=48:band=10"
334
335# Generate the cutlist from the flagged commercials, if requested
336if cut:
337 print "Generating cutlist from comercial flags"
338 cmd = shlex.split("mythutil --gencutlist --chanid {} --starttime {}".format(recording.chanid, recording.starttime.astimezone(pytz.utc).strftime('%Y%m%d%H%M%S')))
339 subprocess.check_call(cmd)
340
341# Create working directory
342fifo_dir = tempfile.mkdtemp(dir=os.path.expanduser('~'))
343
344try:
345
346 # Start mythtranscode outputing into the fifos
347 passthru = ""
348 if audio_format in ['ac3', 'dts', 'mp3']:
349 passthru = "--passthrough"
350 cmdline = "nice mythtranscode --chanid {} --starttime {} --honorcutlist {} --fifodir {} --cleancut".format(recording.chanid, recording.starttime.astimezone(pytz.utc).strftime('%Y%m%d%H%M%S'), passthru, fifo_dir)
351 fifo_proc = subprocess.Popen(cmdline, shell=True, preexec_fn=os.setsid)
352
353 # Wait for the fifos to be established
354 while not (os.path.exists(os.path.join(fifo_dir, 'audout')) and os.path.exists(os.path.join(fifo_dir, 'vidout'))):
355 time.sleep(1)
356
357 # Copy the video into an mkv file using a lossless codec, copying or
358 # recoding the audio as needed
359 if audio_format in ['ac3', 'dts', 'mp3']:
360 audio_in_spec = "-f {}".format(audio_format)
361 audio_out_codec = "copy"
362 else:
363 audio_in_spec = "-f s16le -ac 2"
364 audio_out_codec = "aac -ab 192k -strict experimental"
365 if audio_rate != "0":
366 audio_in_spec += " -ar {}".format(audio_rate)
367 audio_out_codec += " -ar {}".format(audio_rate)
368
369 cmdline = "nice ffmpeg -threads 0 {} -i {} -f rawvideo -top 1 -pix_fmt yuv420p -s {}x{} -r {} -i {} -vcodec libx264 -qp 0 -preset ultrafast -aspect {} {} -acodec {} {}".format(audio_in_spec, os.path.join(fifo_dir, "audout"), video_width, video_height, video_fps, os.path.join(fifo_dir, "vidout"), video_aspect, delogo, audio_out_codec, os.path.join(fifo_dir, "output.mkv"))
370 proc = subprocess.Popen(cmdline, shell=True)
371 proc.wait()
372
373 # If ffmpeg fails, then kill mythtranscode and quit
374 if proc.returncode != 0:
375 print 'ffmpeg finished with rc = {}, terminating'.format(proc.returncode)
376 os.killpg(os.getpgid(fifo_proc.pid), signal.SIGTERM)
377 shutil.rmtree(fifo_dir)
378 sys.exit(_proc.returncode)
379
380 fifo_proc.wait()
381
382 # Delete the cutlist if we generated it
383 if cut:
384 print "Clearing cutlist"
385 cmd = shlex.split("mythutil --clearcutlist --chanid {} --starttime {}".format(recording.chanid, recording.starttime.astimezone(pytz.utc).strftime('%Y%m%d%H%M%S')))
386 subprocess.check_call(cmd)
387
388 # Use HandBrake to recode the video in x264/mp4 format
389 tempfile = tempfile.mktemp(dir=fifo_dir)
390
391 # Best size options for x264:
392 # - SD (4:3 aspect ratio): 320x240, 432x320, 480x360, 544x400, 640x480, 768x576
393 # - HD (16:9 aspect ratio): 432x240, 576x320, 640x360, 720x400, 848x480, 1024x576, 1280x720, 1920x1080
394
395 # Set Handbrake encoding parms
396 if float(video_aspect) > 1.5:
397 handbrake_size = "-w 720"
398 else:
399 handbrake_size = "-w 640"
400
401 if audio_format in ['ac3', 'dts', 'mp3']:
402 handbrake_audio_codec = "copy:{}".format(audio_format)
403 else:
404 handbrake_audio_codec = "copy:aac"
405
406 # Options come from HandBrake's "Normal" profile and x264's "slow" preset
407 cmdline = "nice HandBrakeCLI {} -e x264 -q 20 -a 1 -E {} -B 160 -6 dpl2 -R Auto -D 0.0 -f mp4 --loose-anamorphic -m --encopts b-adapt=2:direct=auto:me=umh:rc-lookahead=50:ref=5:subme=8:level=3.1 --decomb --detelecine -i {} -o {}".format(handbrake_size, handbrake_audio_codec, os.path.join(fifo_dir, "output.mkv"), tempfile)
408 cmd = shlex.split(cmdline)
409 subprocess.check_call(cmd)
410
411 # Normalize the audio, when requested
412 # Implies downsampling to two channel stereo and encoding in AAC
413 if normalize:
414 cmdline = "nice ffmpeg -threads 0 -i {} -vn -acodec pcm_s16le -ac 2 {}".format(tempfile, os.path.join(fifo_dir, "audio.wav"))
415 cmd = shlex.split(cmdline)
416 subprocess.check_call(cmd)
417 cmdline = "normalize {}".format(os.path.join(fifo_dir, "audio.wav"))
418 cmd = shlex.split(cmdline)
419 subprocess.check_call(cmd)
420 cmdline = "nice ffmpeg -threads 0 -i {} -i {} -map 0:0 -map 1:0 -vcodec copy -strict experimental -acodec aac -ab 160k {}".format(tempfile, os.path.join(fifo_dir, "audio.wav"), os.path.join(fifo_dir, "normalized.mp4"))
421 cmd = shlex.split(cmdline)
422 subprocess.check_call(cmd)
423 tempfile = os.path.join(fifo_dir, "normalized.mp4")
424
425 # Move the final MP4 file to the library
426 print "\nMoving video to {}".format(target_file)
427 shutil.move(tempfile, target_file)
428
429 # Create video database entry
430 print "\nInserting video into database"
431 video = Video.fromFilename(target_file.replace('/Video/', '', 1))
432 video.title = recording.title
433 video.subtitle = recording.subtitle
434 video.create()
435
436 # Update database entry with TheTVDB.com metadata, if we got it
437 if meta:
438 video.importMetadata(meta)
439
440 if video.subtitle != meta.subtitle:
441 video.subtitle = meta.subtitle
442
443 # Image files are not imported by importMetadata
444 for image in meta.images:
445 if (image.type in ('screenshot', 'coverart', 'banner')):
446 if (image.type == 'coverart'):
447 key = 'coverfile'
448 else:
449 key = image.type
450 # If we don't already have an image file of this type
451 if (video[key] in (None, '', 'No Cover')):
452 # Get the image file
453 image_file = getImage(image.filename, image.type, image.url, recording.hostname)
454 if image_file != '':
455 # Put the file name into the database
456 video[key] = os.path.basename(image_file)
457
458 # Fill in remaining blanks with local database metadata
459 if not video.plot:
460 video.plot = recording.description
461 if not video.inetref and recording.inetref != '':
462 video.inetref = recording.inetref
463 if not video.year:
464 video.year = recording.year
465 if not video.releasedate:
466 video.releasedate = "{:%Y-%m-%d}".format(recording.starttime.astimezone(pytz.utc))
467 if not video.season or video.season == '0':
468 video.season = recording.season
469 if not video.episode or video.episode == '0':
470 video.episode = recording.episode
471
472 # Calculate the file hash
473 video.hash = hashFile(target_file)
474
475 # Set host name
476 video.host = recording.hostname
477
478 # Update the database entry with these changes
479 video.update()
480
481finally:
482 # Delete working directory
483 print 'Deleting {}'.format(fifo_dir)
484 shutil.rmtree(fifo_dir)
485
486print "\nAll done. That all took {}".format(elapsedTime(elapsed_start))