diff --git a/backend/config_parser.py b/backend/config_parser.py index ba7faea..af9e035 100644 --- a/backend/config_parser.py +++ b/backend/config_parser.py @@ -15,6 +15,10 @@ from global_funcs import * class NeedNVR(Exception): pass +class ImageNoCreated(Exception): + def __init__(self, msg): + pass + class Recorder: loop = asyncio.get_event_loop() def __init__(self, address, port, username, password, name = "", index = 0, name_overrides = {}): @@ -155,6 +159,7 @@ class TranscodeTools: def __init__(self, tools_directory, transcode_directory, hide_checks = True, delete_temporary_files = False) -> None: self.delete_temporary_files = delete_temporary_files self.logger = create_logger(TranscodeTools.__name__) + debug(self.logger) self.hide_checks = hide_checks self.tools_directory = tools_directory self.transcode_directory = transcode_directory @@ -183,6 +188,20 @@ class TranscodeTools: self.logger.error("Cannot enabled transcode tools, have a errors on init, run config_parser with --no-hide-check parameters to more info") self.logger.info("Transcode tools " + "enabled" if self.enabled else "disabled") + self.queue_encoding = asyncio.Queue() + self.queue_images = asyncio.Queue() + + async def queue_encoding_task(self): + self.logger.info("Start encoding queue") + while True: + await self.queue_encoding.get() + await asyncio.sleep(1) + + async def queue_images_task(self): + self.logger.info("Start images queue") + while True: + await self.queue_images.get() + await asyncio.sleep(1) @property def check_exists_needed_files(self): @@ -269,7 +288,7 @@ class TranscodeTools: if os.path.exists(out_file): return out_file else: - raise Exception(f"{out_file} not be created") + raise ImageNoCreated(f"{out_file} not be created") def deleteFile(self, source_file): try: @@ -277,6 +296,12 @@ class TranscodeTools: except: pass + async def processing_preview_task(self, file:File, nvr: NVR, ext = "webp", preview_pre_bytes = 1024 * 512): + async def task(file, nvr, ext, preview_pre_bytes): + await self.processing_preview(file, nvr, ext, preview_pre_bytes) + + self.queue_images.put_nowait(asyncio.create_task(task(file, nvr, ext, preview_pre_bytes))) + async def processing_preview(self, file:File, nvr: NVR, ext = "webp", preview_pre_bytes = 1024 * 512): raw_file = file.previewPath(self.transcode_directory) preview_file = f"{raw_file}.{ext}" @@ -303,6 +328,7 @@ class TranscodeTools: if not os.path.exists(raw_file) or os.path.getsize(raw_file) == 0: try: + await nvr.login() async with asyncio.timeout(10): self.logger.info(f"download new preview {preview_file}") async with aiofiles.open(raw_file, "wb") as raw: @@ -319,14 +345,24 @@ class TranscodeTools: raw_file_avi = await self.h264toavi(raw_file) if self.delete_temporary_files: self.deleteFile(raw_file) - return await self.anytoimage(raw_file_avi, preview_file, self.delete_temporary_files) - async def processing_safe(self, status: TranscodeStatus, file:File, nvr: NVR, reCreate:bool = False): try: - await self.processing_mp4(status, file, nvr, reCreate) - except Exception as e: - traceback.print_exc() - self.statuses[status.b64].error = str(e) + return await self.anytoimage(raw_file_avi, preview_file, self.delete_temporary_files) + except ImageNoCreated: + clear_from_storage() + self.logger.warning("Redownload with more size") + return await self.processing_preview(file, nvr, ext, preview_pre_bytes * 2) + + async def processing_safe(self, status: TranscodeStatus, file:File, nvr: NVR, reCreate:bool = False): + async def task(status: TranscodeStatus, file:File, nvr: NVR, reCreate): + try: + await self.processing_mp4(status, file, nvr, reCreate) + except Exception as e: + traceback.print_exc() + self.statuses[status.b64].error = str(e) + + self.logger.info(f"Start enconding {file.filename_cleared}") + self.queue_encoding.put_nowait(asyncio.create_task(task(status = status, file = file, nvr = nvr, reCreate = reCreate))) async def processing_mp4(self, status: TranscodeStatus, file:File, nvr: NVR, reCreate:bool = False): raw_file = file.localPath(self.transcode_directory) @@ -334,16 +370,18 @@ class TranscodeTools: mp4_file = os.path.join(f"{raw_file}.avi.mp4") if os.path.exists(mp4_file) and os.path.getsize(mp4_file) != 0: - nvr.logout() - self.statuses[status.b64].outFile = mp4_file - self.statuses[status.b64].done = True - self.statuses[status.b64].outSize = os.path.getsize(mp4_file) - return + if not reCreate: + nvr.logout() + self.statuses[status.b64].outFile = mp4_file + self.statuses[status.b64].done = True + self.statuses[status.b64].outSize = os.path.getsize(mp4_file) + return - if not os.path.exists(raw_file) or os.path.getsize(raw_file) != file.size: + if not os.path.exists(raw_file) or os.path.getsize(raw_file) != file.size or reCreate: self.logger.debug(f"save raw file to {raw_file}") async with aiofiles.open(raw_file, "wb") as raw: self.statuses[status.b64].total_h264_bytes = file.size + await nvr.login() async for chunk in nvr.stream_file(file): self.statuses[status.b64].downloaded_h264_bytes += len(chunk) self.statuses[status.b64].h264 = round(100 * self.statuses[status.b64].downloaded_h264_bytes / self.statuses[status.b64].total_h264_bytes) diff --git a/backend/global_funcs.py b/backend/global_funcs.py index 5c7ebfa..3d4cb08 100644 --- a/backend/global_funcs.py +++ b/backend/global_funcs.py @@ -38,3 +38,6 @@ def create_logger(t, level:int = logging.INFO, format=None): ch.setFormatter(formatter) logger.addHandler(ch) return logger + +def debug(logger): + logger.setLevel(logging.DEBUG) diff --git a/backend/server.py b/backend/server.py index 26d0f68..22772c9 100644 --- a/backend/server.py +++ b/backend/server.py @@ -42,6 +42,9 @@ class Server: self.logger.info(f"{self.config.recorders[i]} channels count: {self.config.recorders[i].channels}") await self.go2rtc.start_go2rtc(self.config.recorders) + asyncio.create_task(self.config.transcode_tools.queue_images_task()) + asyncio.create_task(self.config.transcode_tools.queue_encoding_task()) + def setup_middleware(self): if len(self.config.auth) != 0: self.logger.info("auth is enabled") @@ -152,21 +155,21 @@ class Server: return {"ok":False, "error":e} @self.app.get(self.API_BASE_REF + "/transcode/status/{recorder_index}") - async def getTranscodeStatus(response: Response, recorder_index:int, b64:str, background_tasks: BackgroundTasks): + async def getTranscodeStatus(response: Response, recorder_index:int, b64:str, background_tasks: BackgroundTasks, recreate:bool = False): try: if len(b64) == 0: response.status_code = 404 return "" if b64 in self.config.transcode_tools.statuses: - return {"ok":True, "data":self.config.transcode_tools.statuses[b64]} + if not recreate: + return {"ok":True, "data":self.config.transcode_tools.statuses[b64]} nvr:NVR = self.config.getRecorder(recorder_index).nvr - await nvr.login() file: File = File.from_b64(b64 + "==") self.config.transcode_tools.statuses[b64] = TranscodeStatus(b64) - background_tasks.add_task(self.config.transcode_tools.processing_safe, status = self.config.transcode_tools.statuses[b64], file = file, nvr = nvr) + background_tasks.add_task(self.config.transcode_tools.processing_safe, status = self.config.transcode_tools.statuses[b64], file = file, nvr = nvr, reCreate = recreate) return {"ok":True, "data":self.config.transcode_tools.statuses[b64]} except Exception as e: traceback.print_exc() @@ -229,19 +232,23 @@ class Server: traceback.print_exc() response.status_code = 400 return {"ok":False, "error":e} + + @self.app.get(self.API_BASE_REF + "/preview") + async def getNullPreview(): + return FileResponse("./assets/loading.webp") + @self.app.get(self.API_BASE_REF + "/preview/{recorder_index}") - async def getTranscodePreview(response: Response, recorder_index:int, b64:str, background_tasks: BackgroundTasks): + async def getTranscodePreview(response: Response, recorder_index:int, b64:str, background_tasks: BackgroundTasks, status:bool = False): try: if len(b64) == 0: response.status_code = 404 - return "" + return "" if not status else {"status":-1} file: File = File.from_b64(b64 + "==") async def loadPreview(file): nvr:NVR = self.config.getRecorder(recorder_index).nvr - await nvr.login() - await self.config.transcode_tools.processing_preview(file, nvr, "webp") + await self.config.transcode_tools.processing_preview_task(file, nvr, "webp") try: preview = await self.config.transcode_tools.processing_preview(file, None, "webp") @@ -254,10 +261,10 @@ class Server: raise NeedNVR headers.update({"Content-Disposition": f'attachment; filename="preview.webp"'}) - return FileResponse(preview, media_type="application/octet-stream", headers=headers) + return FileResponse(preview, media_type="application/octet-stream", headers=headers) if not status else {"status":1} except NeedNVR: background_tasks.add_task(loadPreview, file = file) - return FileResponse("./assets/loading.webp") + return FileResponse("./assets/loading.webp") if not status else {"status":0} except Exception as e: traceback.print_exc() diff --git a/frontend/ang_dvrip/src/app/components/history/history.component.html b/frontend/ang_dvrip/src/app/components/history/history.component.html index 0f75c79..2ba8782 100644 --- a/frontend/ang_dvrip/src/app/components/history/history.component.html +++ b/frontend/ang_dvrip/src/app/components/history/history.component.html @@ -35,7 +35,7 @@ mat-card-image style="cursor: pointer" (click)="openTransCodeDialog(element)" - [src]="'api/dvrip/preview/'+recorder_index+'?b64='+element.b64+'×tamp='+this.baseUtils.getTime()" + [src]="element.preview" alt="Превью недоступно"> diff --git a/frontend/ang_dvrip/src/app/components/history/history.component.ts b/frontend/ang_dvrip/src/app/components/history/history.component.ts index 5215374..f532122 100644 --- a/frontend/ang_dvrip/src/app/components/history/history.component.ts +++ b/frontend/ang_dvrip/src/app/components/history/history.component.ts @@ -39,6 +39,9 @@ export class HistoryComponent implements OnInit { this.http.get(`api/dvrip/history/${params.get('recorderId')}/${params.get('channelId')}/${this.selected_stream}?start_date=${this.baseUtils.prepareDate(this.start_date, true)}&end_date=${this.baseUtils.prepareDate(this.end_date, false)}`) .subscribe((a:any) => { this.dataSource = a['data']; + for (let element of this.dataSource) { + this.getPreviewImg(element); + } }) } @@ -69,4 +72,33 @@ export class HistoryComponent implements OnInit { } }); } + + getPreviewImg(element:DVRFILE) { + this.http.get(`api/dvrip/preview/${this.recorder_index}?b64=${element.b64}&status=${true}×tamp=${this.baseUtils.getTime()}`).subscribe((result:any) => { + console.log(result); + switch (result.status){ + case -1: { + element.preview = "api/dvrip/preview"; + break; + } + case 0: { + element.preview = "api/dvrip/preview"; + if (element.preview_checker == null) { + element.preview_checker = setInterval(() => { + this.getPreviewImg(element) + }, 1000); + } + + break; + } + case 1: { + element.preview = `api/dvrip/preview/${this.recorder_index}?b64=${element.b64}`; + if (element.preview_checker != null) { + clearInterval(element.preview_checker); + } + break; + } + } + }) + } } diff --git a/frontend/ang_dvrip/src/app/entities/DVRFILE.ts b/frontend/ang_dvrip/src/app/entities/DVRFILE.ts index 096a2d9..e55e9e3 100644 --- a/frontend/ang_dvrip/src/app/entities/DVRFILE.ts +++ b/frontend/ang_dvrip/src/app/entities/DVRFILE.ts @@ -4,5 +4,7 @@ export default interface DVRFILE { b64:string, converted:boolean, processing:boolean|null, - date:string + date:string, + preview:string, + preview_checker:any } diff --git a/frontend/ang_dvrip/src/app/modals/transcode-modal/transcode-modal.component.html b/frontend/ang_dvrip/src/app/modals/transcode-modal/transcode-modal.component.html index d8134c2..859d8c0 100644 --- a/frontend/ang_dvrip/src/app/modals/transcode-modal/transcode-modal.component.html +++ b/frontend/ang_dvrip/src/app/modals/transcode-modal/transcode-modal.component.html @@ -24,4 +24,5 @@ + diff --git a/frontend/ang_dvrip/src/app/modals/transcode-modal/transcode-modal.component.ts b/frontend/ang_dvrip/src/app/modals/transcode-modal/transcode-modal.component.ts index 6019406..6bf7898 100644 --- a/frontend/ang_dvrip/src/app/modals/transcode-modal/transcode-modal.component.ts +++ b/frontend/ang_dvrip/src/app/modals/transcode-modal/transcode-modal.component.ts @@ -35,15 +35,17 @@ export class TranscodeModalComponent implements OnInit { this.dialogRef.close(this.status.done); } - getStatus() { - this.client.get(`api/dvrip/transcode/status/${this.data.recorder_index}?b64=${this.data.b64}`) + getStatus(reCreate:boolean = false) { + this.client.get(`api/dvrip/transcode/status/${this.data.recorder_index}?b64=${this.data.b64}&recreate=${reCreate}`) .subscribe((data:any) => { this.status = new TranscodeStatus().fromData(data["data"]); this.loading = false; if (this.status.done) clearInterval(this.interval); console.log(this.status); - }) + }, error => clearInterval(this.interval)) + if (reCreate) + this.ngOnInit(); } getMP4(b64:string) {