fix: 修复若干问题

isc_video_record/db/mysql.py

+import json
 from contextlib import contextmanager
 from functools import wraps
 
@@ -128,7 +129,9 @@ def insert_video_info(cursor, conn, db_table, device_code, start_time, end_time,
 
 @query()
 def update_video_info(cursor, conn, db_table, video_id, status,
-                      file_name=None, video_url=None, video_resolution=None, recovered_time=None):
+                      file_name=None, video_url=None,
+                      video_resolution=None, recovered_time=None, retry_info=None,
+                      next_retry_time=None, remark=None):
     sql = '''
         update {}
         set status = %s, update_time = now() {}
@@ -139,6 +142,14 @@ def update_video_info(cursor, conn, db_table, video_id, status,
         sub_set += ', file_name = "{}", video_url="{}", video_resolution="{}", recovered_time="{}",' \
                    'expired_time=date_add(now(),interval 31 day)'\
             .format(file_name, video_url, video_resolution, recovered_time)
+    if retry_info:
+        sub_set += ', retry_info=\'{}\''.format(json.dumps(retry_info, ensure_ascii=False))
+    if next_retry_time:
+        sub_set += ', next_retry_time="{}"'.format(next_retry_time)
+    if remark:
+        sub_set += ', remark="{}"'.format(remark)
+
+    log.info(sql.format(db_table, sub_set), status, video_id)
     cursor.execute(sql.format(db_table, sub_set), [status, video_id])
     conn.commit()
 
@@ -147,7 +158,7 @@ def update_video_info(cursor, conn, db_table, video_id, status,
 def get_untreated_events(cursor, conn, db_table, camera_code, status=3, **kw):
 
     if 'retry' in kw:
-        sub_str = 'status in (2, 3) and update_time > date_sub(now(), interval 12 hour)'
+        sub_str = '(status = 2 and next_retry_time < now()) or status = 3'
     else:
         sub_str = 'status = {}'.format(status)
 
@@ -159,7 +170,8 @@ def get_untreated_events(cursor, conn, db_table, camera_code, status=3, **kw):
             status,
             recovered_time,
             video_url,
-            file_name
+            file_name,
+            retry_info, next_retry_time, remark
         from {}
         where device_code = %s
           and create_time > date_sub(now(), interval 7 day)
@@ -169,3 +181,18 @@ def get_untreated_events(cursor, conn, db_table, camera_code, status=3, **kw):
 
     cursor.execute(sql, [camera_code])
     return cursor.fetchall()
+
+
+@query(cursor_dict=True)
+def get_events_by_time(cursor, conn, db_table, camera_code, start_time, end_time):
+    sql = '''
+        select *
+        from {}
+        where device_code = %s
+            and start_time >= %s
+            and end_time < %s
+        order by start_time
+    '''.format(db_table)
+    cursor.execute(sql, [camera_code, start_time, end_time])
+    return cursor.fetchall()
+

isc_video_record/recorder.py

@@ -8,6 +8,7 @@ import pytz
 import dynaconf
 
 from datetime import timedelta, datetime
+from redis.exceptions import ConnectionError
 from intelab_python_sdk.logger import log_init, log
 from intelab_python_sdk.ffmpeg.ffmpeg_concat import concat
 
@@ -36,6 +37,7 @@ class StreamRecorder:
 
     @staticmethod
     def set_nx(pipe, camera_key, thread_id, ex):
+        pipe.pipeline()
         res = pipe.set(camera_key, thread_id, nx=True, ex=int(ex))
         return res
 
@@ -110,8 +112,7 @@ class StreamRecorder:
     def process_message(self, pipe, camera_key, thread_id, body):
         ack = False
 
-        events = mysql.get_untreated_events(body['db_table'], body['camera_code'],
-                                            retry='status=2')
+        events = mysql.get_untreated_events(body['db_table'], body['camera_code'], retry=True)
         log.info('Thread_id:%s: events count: %s', thread_id, len(events))
 
         pipe.incr(PROCESSING_TOTAL_KEY)  # 当前录制进程数+1
@@ -126,13 +127,14 @@ class StreamRecorder:
                 break
         else:
             ack = True
+        try:
             pipe.decr(PROCESSING_TOTAL_KEY)  # 本次录制结束进程数-1
+        except ConnectionError:
+            with redis_connect() as pipe:
+                pipe.decr(PROCESSING_TOTAL_KEY)
         return ack
 
     def recording(self, thread_id, db_table, event):
-        # TODO 录制失败的视频重试和续录
-        if event['status'] == 2:
-            log.info('下载失败重试任务%s', event['file_name'])
 
         t1 = time.time()
         record_result = self.recorder(
@@ -144,8 +146,8 @@ class StreamRecorder:
         t2 = time.time()
 
         video_info, error_log = get_video_duration(record_result['file_name'])
-        log.info('Thread_id:%s: Time consuming: %s, duration: %s, size: %sM',
-                 thread_id,
+        log.info('Thread_id:%s:%s: Time consuming: %s, duration: %s, size: %sM',
+                 thread_id, event['camera_code'],
                  round(t2 - t1, 2), time_to_seconds(video_info['duration']),
                  video_info['size'])
         url = ''
@@ -154,13 +156,38 @@ class StreamRecorder:
             url = aliyun_oss.oss_upload_file('isc_record/' + file_name,
                                              record_result['file_name'])
             status = 1 if record_result['is_completed'] else 2
+            # TODO 失败的续录
             mysql.update_video_info(db_table, event['video_id'], status,
                                     file_name=file_name, video_url=url, video_resolution=video_info['resolution'],
                                     recovered_time=record_result['recovered_time'].astimezone(pytz.utc))
             os.remove(record_result['file_name'])
         else:
+            now = datetime.now()
+            if not event.get('retry_info'):
+                event['retry_info'] = '[]'
+
+            event['retry_info'] = json.loads(event['retry_info'])
+            retry_count = len(event['retry_info'])
+            next_retry_time = remark = None
+
+            event['retry_info'].append({
+                'retry_count': retry_count + 1,
+                'retry_time': now.strftime('%Y-%m-%d %H:%M:%S'),
+                'result': 'failed'
+            })
+            if retry_count < 5:
                 # 无有效视频文件，标记事件状态 status = 2
-            mysql.update_video_info(db_table, event['video_id'], 2)
+                status = 2
+                next_retry_time = now + timedelta(hours=3)
+                remark = (event.get('remark') or 'start') + '+failed'
+            else:
+                status = 4  # 状态4: 不再重试！
+                remark = event['remark'] + '+failed+end.'
+
+            mysql.update_video_info(db_table, event['video_id'], status,
+                                    retry_info=event['retry_info'],
+                                    next_retry_time=next_retry_time,
+                                    remark=remark)
 
         log.info('video_info: %s, url: %s, video_id: %s.%s, time: %s',
                  video_info, url, db_table,
@@ -180,7 +207,7 @@ class StreamRecorder:
             IntelabApiHelper.iso_format(start_time),
             IntelabApiHelper.iso_format(end_time)
         )
-        log.info('thread_id:%s: playback: %s', thread_id, playback_urls)
+        log.info('thread_id:%s:%s: playback: %s', thread_id, camera_code, playback_urls)
         file_name = os.path.join(video_path, 'ISC_{}_{}_{}.mp4'.format(
             camera_code,
             start_time.astimezone(pytz.utc).strftime('%Y%m%dT%H%M%S'),
@@ -195,15 +222,17 @@ class StreamRecorder:
         part_num = retry_count = 0
         is_completed = False
         part_files_set = set()
-        while retry_count < 6:
+        while retry_count < 3:
             # 重试六次
             retry_count += 1
             complete_duration = (end_time - start_time).total_seconds()
 
             file_info, _ = StreamRecorder.stream_record(
-                playback_stream['stream_url'], start_time, end_time)
+                playback_stream['stream_url'], start_time, end_time, camera_code)
             file_duration = time_to_seconds(file_info['duration'])
             if not os.path.isfile(file_info['file_name']):
+                log.info('当前录制无文件输出:%s, 重试计数: %s', camera_code, retry_count)
+                time.sleep(1)
                 continue
 
             if file_duration < complete_duration - 2:
@@ -231,7 +260,8 @@ class StreamRecorder:
             concat(part_files, file_name, removed=True)
         elif len(part_files) == 1:
             shutil.move(part_files[0], file_name)
-        log.info('The download is complete, file %s', file_name)
+        log.info('thread_id:%s:%s: The download is complete, file %s',
+                 thread_id, camera_code, file_name)
         return {
             'file_name': file_name,
             'is_completed': is_completed,
@@ -240,7 +270,7 @@ class StreamRecorder:
         }
 
     @staticmethod
-    def stream_record(stream, start_time, end_time):
+    def stream_record(stream, start_time, end_time, camera_code):
         start_time = start_time.strftime('%Y%m%dT%H%M%S')
         end_time = end_time.strftime('%Y%m%dT%H%M%S')
 
@@ -252,8 +282,8 @@ class StreamRecorder:
                                                                 end_time, stream['extra_args'])
         file_name = os.path.join(
             video_path, 'rtmp_{}_{}.mp4'.format(start_time, end_time))
-        log.info('stream_url: %s', stream_url)
-        record_thread(stream_url, file_name)
+        log.info('{} stream_url: %s', camera_code, stream_url)
+        record_thread(stream_url, file_name, thread_name=camera_code)
         return get_video_duration(file_name)
 
 

isc_video_record/utils/excel_utils.py

+import openpyxl
+
+
+def gen_excel(rows, headers, columns, file_name, title='sheet'):
+    new = openpyxl.Workbook()
+    sheet = new.active
+    sheet.title = title
+    for index, value in enumerate(headers):
+        _ = sheet.cell(row=1, column=index+1, value=u'%s' % value)
+
+    for row, target in enumerate(rows):
+        for col, key in enumerate(columns):
+            _ = sheet.cell(row=row+2, column=col + 1,
+                           value=u'%s' % target[key])
+    newworkbook = new.save(file_name)
+    return newworkbook

isc_video_record/utils/record_utils.py

@@ -4,7 +4,7 @@ import subprocess
 from intelab_python_sdk.logger import log
 
 
-def record_thread(stream_url, out_file):
+def record_thread(stream_url, out_file, thread_name='ffmpeg-log'):
 
     cmd = [
         'ffmpeg',
@@ -40,9 +40,9 @@ def record_thread(stream_url, out_file):
                 log_buffer = ''
 
             if log_buffer and 'error' in log_buffer:
-                log.error('ffmpeg-log:error: %s', log_buffer.strip())
+                log.error('%s:error: %s', thread_name, log_buffer.strip())
             elif log_buffer:
-                log.debug('ffmpeg-log: %s', log_buffer.strip())
+                log.debug('%s: %s', thread_name, log_buffer.strip())
 
             log_buffer = stdout
 

setup.py

@@ -17,12 +17,13 @@ requires = [
     'mysql-connector',
     'retrying',
     'oss2',
-    'apscheduler'
+    'apscheduler',
+    'openpyxl'
 ]
 
 setuptools.setup(
     name='isc-video-record',
-    version='1.0.0a9',
+    version='1.0.0a13',
     description='ISC motion detection playback video stream recording service.',
     long_description=long_description,
     long_description_content_type='text/markdown',

tests/test_clean_processing.py

+from isc_video_record.db.redis import redis_connect
+from isc_video_record.const import PROCESSING_CAMERA_KEY, PROCESSING_TOTAL_KEY
+
+
+with redis_connect() as pipe:
+    processing_keys = pipe.keys(PROCESSING_CAMERA_KEY.format('*'))
+    for key in processing_keys:
+        print('清除任务key:', key)
+        pipe.delete(key)
+    print('正在录制中摄像头数量:', pipe.get(PROCESSING_TOTAL_KEY))
+    pipe.delete(PROCESSING_TOTAL_KEY)

tests/test_redis_setnx.py

+import time
+import threading
+from isc_video_record.db.redis import redis_connect
+
+
+def test_setnx():
+
+    with redis_connect() as pipe:
+        res = pipe.set('b', '1', nx=True, ex=60)
+        time.sleep(2)
+        print('设置结果', res)
+
+
+ths = []
+for i in range(100):
+    t = threading.Thread(target=test_setnx, name='thread_{}'.format(i))
+    t.start()
+    ths.append(t)
+
+
+for t in ths:
+    t.join()