Add disabled deploy targets and FFmpeg DL v2

Skip disabled deployment entries across hosting workflows. Add a v2 download route with server-to-server progress relaying, move Redis and offline-store utilities into shared packages, and enable audio re-encoding for input-phase FFmpeg jobs.

src/lib/ffmpeg-v2.ts

@@ -605,7 +605,7 @@ async function F_build_ffmpeg_flags_t1(arg0: {
   // const re_encode_va = !IS_RENDER_COM();
 
   if (phase_input) must_re_encode = false;
-  if (phase_input) re_encode_a = false;
+  if (phase_input) re_encode_a = true;
 
   flags.push("-map", "0:v");
   flags.push("-map", "0:a?");

src/lib/offline-store.ts

@@ -1,72 +0,0 @@
-// Simple Store using Map
-class OfflineStore {
-  state: Map<any, any>;
-  subscribers: Set<(state: any) => void>;
-
-  constructor() {
-    this.state = new Map();
-    this.subscribers = new Set();
-  }
-
-  // Get a value by key
-  get(key: string) {
-    return this.state.get(key);
-  }
-
-  // Set a value by key and notify subscribers
-  set(key: string, value: any) {
-    this.state.set(key, value);
-    this.notifySubscribers();
-  }
-
-  // Delete a value by key and notify subscribers
-  delete(key: string) {
-    this.state.delete(key);
-    this.notifySubscribers();
-  }
-
-  // Clear the entire store and notify subscribers
-  clear() {
-    this.state.clear();
-    this.notifySubscribers();
-  }
-
-  // Subscribe to store changes
-  subscribe(callback: (state: any) => void) {
-    this.subscribers.add(callback);
-    return () => this.subscribers.delete(callback);
-  }
-
-  // Notify all subscribers
-  notifySubscribers() {
-    this.subscribers.forEach((callback) => {
-      return callback(this.state);
-    });
-  }
-}
-
-const offline_store = new OfflineStore();
-
-export { offline_store };
-
-// Example usage
-// Subscribe to changes
-// const unsubscribe = store.subscribe((state: any[]) => {
-//   console.log("Store updated:", Array.from(state.entries()));
-// });
-
-// Set values
-// store.set("user", { name: "Alice", age: 30 });
-// store.set("theme", "dark");
-
-// Get a value
-// console.log(store.get("user")); // { name: "Alice", age: 30 }
-
-// Delete a value
-// store.delete("theme");
-
-// Clear the store
-// store.clear();
-
-// Unsubscribe
-// unsubscribe();

src/lib/upstash.ts

@@ -1,18 +0,0 @@
-import { Redis } from "@upstash/redis";
-import { NS_runtime } from "@x/types/env.runtime";
-
-const UPSTASH_REDIS_REST_URL = () =>
-  NS_runtime.ENV_VAR_DB?.UPSTASH_REDIS_REST_URL ||
-  process.env.UPSTASH_REDIS_REST_URL;
-const UPSTASH_REDIS_REST_TOKEN = () =>
-  NS_runtime.ENV_VAR_DB?.UPSTASH_REDIS_REST_TOKEN ||
-  process.env.UPSTASH_REDIS_REST_TOKEN;
-
-const redis = () => {
-  return new Redis({
-    url: UPSTASH_REDIS_REST_URL(),
-    token: UPSTASH_REDIS_REST_TOKEN(),
-  });
-};
-
-export { redis };

src/sub-router/probe.ts

@@ -8,10 +8,8 @@ import UserAgent from "user-agents";
 import { NS_promise } from "@x/core/client/promise.ts";
 import { NS_crypto_client } from "@x/core/client/crypto.ts";
 import { NS_unpack_proxima } from "@x/core/server/proxima/unpack.ts";
-import { redis } from "../lib/upstash";
-import { offline_store } from "../lib/offline-store";
+import { offline_store } from "@x/ff-dl/offline-store";
 import { estimateHlsSize } from "./hls-size";
-import { NS_scrape_post_processing_helper } from "@x/post-process/helper";
 import {
   NS_probe_validator_v2,
   type T_probe_playback,
@@ -20,6 +18,7 @@ import {
 import { NS_ff } from "@x/ff-dl/helper";
 import { NS_ffmpeg_v2 } from "../lib/ffmpeg-v2";
 import { NS_error } from "@x/core/client/error.ts";
+import { redis } from "@x/post-process/cache";
 
 const APP_ACCOUNT_ABC = () =>
   NS_runtime.ENV_VAR_DB?.APP_ACCOUNT_ABC || process.env.APP_ACCOUNT_ABC;
@@ -544,7 +543,7 @@ const sub_router = new Elysia({ prefix: "/ffmpeg" })
         if (APP_ENV() === "dev") {
           offline_store.set(key, delivery_payload);
         } else {
-          await redis().set(key, delivery_payload, {
+          await redis()?.set(key, delivery_payload, {
             ex: ONE_MONTH_IN_SECONDS
           });
         }

src/sub-router/showtime.ts

@@ -4,14 +4,13 @@ import {
   IS_RENDER_COM,
   NS_runtime
 } from "@x/types/env.runtime";
-import { redis } from "../lib/upstash";
 import { spawn } from "child_process";
 import { type Context, Elysia, t } from "elysia";
 import path from "node:path";
 import { PassThrough, Transform } from "node:stream";
 import { NS_ffmpeg_v2 } from "../lib/ffmpeg-v2";
 import { NS_misc } from "../lib/misc";
-import { offline_store } from "../lib/offline-store";
+import { offline_store } from "@x/ff-dl/offline-store";
 import { NS_promise } from "@x/core/client/promise.ts";
 import node_fs from "node:fs";
 import { NS_pack_proxima } from "@x/core/server/proxima/pack.ts";
@@ -32,6 +31,8 @@ import {
 import type { T_delivery_payload, T_probe_playback } from "@x/ff-dl/types";
 import { NS_ff } from "@x/ff-dl/helper";
 import { homedir } from "node:os";
+import { redis } from "@x/post-process/cache";
+import { NS_crypto_client } from "@x/core/client/crypto.ts";
 
 const APP_ACCOUNT_ABC = () =>
   NS_runtime.ENV_VAR_DB?.APP_ACCOUNT_ABC || process.env.APP_ACCOUNT_ABC;
@@ -55,97 +56,150 @@ const is_dev = (() => {
   return APP_ENV() === "dev";
 })();
 
-const sub_router = new Elysia({ prefix: "/dlV1" }).get(
-  "/delivery/:data_key",
-  async (ctx) => {
-    const { params, query, request } = ctx;
-    let { data_key } = params;
-    let { ops } = query;
+const sub_router = new Elysia({ prefix: "/dlV1" })
 
-    const REQ0 = new URL(request.url);
-    const REQ_ORIGIN = (ctx as any).publicOrigin || REQ0.origin;
+  .post(
+    "/s2s",
+    async (ctx) => {
+      const { body, request } = ctx;
+      const { job_id, data_key, status, info } = body;
 
-    const metadata = await (async () => {
-      try {
-        if (is_dev) {
-          const _ = offline_store.get(data_key);
+      const REQ0 = new URL(request.url);
+      const REQ_ORIGIN = (ctx as any).publicOrigin || REQ0.origin;
 
-          if (_) return _;
+      record_task_info({
+        job_id,
+        status: status as any,
+        info
+      });
+
+      relay_progress_info_ws({
+        channel: data_key,
+        payload: {
+          channel: data_key,
+          status,
+          data: info
         }
+      });
+    },
+    {
+      body: t.Object({
+        job_id: t.String({
+          minLength: 1,
+          error() {
+            return "Invalid job id";
+          }
+        }),
+        status: t.String({
+          minLength: 1,
+          error() {
+            return "Invalidstatus";
+          }
+        }),
+        data_key: t.String({
+          minLength: 1,
+          error() {
+            return "Invalid data key";
+          }
+        }),
+        info: t.Record(t.String(), t.Any(), {
+          error: "info must be a valid object"
+        })
+      })
+    }
+  )
 
-        const _ = await redis().get(data_key);
+  .get(
+    "/delivery/:data_key",
+    async (ctx) => {
+      const { params, query, request } = ctx;
+      let { data_key } = params;
+      let { ops } = query;
 
-        return _;
-      } catch {
-        return;
-      }
-    })();
+      const REQ0 = new URL(request.url);
+      const REQ_ORIGIN = (ctx as any).publicOrigin || REQ0.origin;
 
-    if (!metadata) {
-      ctx.set.status = 404;
-      return "Metadata not found";
-    }
+      const metadata = await (async () => {
+        try {
+          if (is_dev) {
+            const _ = offline_store.get(data_key);
 
-    if (ops === "raw" || ops === "decode") {
-      return metadata;
-    }
+            if (_) return _;
+          }
 
-    const [E_, R_] = await NS_promise.F_catch_promise({
-      promise: handle_data({
-        data_key,
-        metadata,
-        REQ0,
-        REQ_ORIGIN,
-        ctx: ctx as any,
-        ops
-      })
-    });
+          const _ = await redis()?.get(data_key);
+
+          return _;
+        } catch {
+          return;
+        }
+      })();
+
+      if (!metadata) {
+        ctx.set.status = 404;
+        return "Metadata not found";
+      }
 
-    if (E_) {
-      const { message, status } = NS_error.F_collect_error_status_n_message({
-        ERR: E_
+      if (ops === "raw" || ops === "decode") {
+        return metadata;
+      }
+
+      const [E_, R_] = await NS_promise.F_catch_promise({
+        promise: handle_data({
+          data_key,
+          metadata,
+          REQ0,
+          REQ_ORIGIN,
+          ctx: ctx as any,
+          ops
+        })
       });
 
-      return ctx.status(status, message);
-    }
+      if (E_) {
+        const { message, status } = NS_error.F_collect_error_status_n_message({
+          ERR: E_
+        });
 
-    return R_;
-  },
-  {
-    params: t.Object({
-      data_key: t.String({
-        minLength: 1,
-        error() {
-          return "Invalid process data, Expected string";
-        }
-      })
-    }),
-    query: t.Object({
-      raw: t.Optional(
-        t.Number({
-          minimum: 0,
-          maximum: 1,
-          error: () => "Invalid raw, Expected binary boolean"
+        return ctx.status(status, message);
+      }
+
+      return R_;
+    },
+    {
+      params: t.Object({
+        data_key: t.String({
+          minLength: 1,
+          error() {
+            return "Invalid process data, Expected string";
+          }
         })
-      ),
-      ops: t.Optional(
-        t.Union(
-          [
-            t.Literal("cmd"),
-            t.Literal("raw"),
-            t.Literal("decode"),
-            t.Literal("jc")
-          ],
-          {
-            error() {
-              return "Invalid operation, must be one of: cmd, raw, decode";
+      }),
+      query: t.Object({
+        raw: t.Optional(
+          t.Number({
+            minimum: 0,
+            maximum: 1,
+            error: () => "Invalid raw, Expected binary boolean"
+          })
+        ),
+        ops: t.Optional(
+          t.Union(
+            [
+              t.Literal("cmd"),
+              t.Literal("raw"),
+              t.Literal("decode"),
+              t.Literal("jc")
+            ],
+            {
+              error() {
+                return "Invalid operation, must be one of: cmd, raw, decode";
+              }
             }
-          }
+          )
         )
-      )
-    })
-  }
-);
+      })
+    }
+  );
 
 export const dlV1Router = sub_router;
 export const dlV1RouterHandlePayload = handle_data;
@@ -176,6 +230,8 @@ async function handle_data(arg0: {
     station
   } = metadata as T_delivery_payload;
 
+  metadata.s2s_progress = (() => new URL("/dlV1/s2s", REQ_ORIGIN).toString())();
+
   const is_download_mode = delivery_mode === "download";
   const is_stream_mode = delivery_mode === "stream";
 
@@ -266,6 +322,20 @@ async function handle_data(arg0: {
       _
     );
 
+    const [E_, R_] = await NS_promise.F_catch_promise({
+      promise: (async () => {
+        return NS_crypto_client.F_encode_unicode_v1({
+          data_payload: metadata,
+          url_safe: true,
+          encode: "hex"
+        });
+      })()
+    });
+
+    if (R_) {
+      __.searchParams.set("data_key", `${R_}`);
+    }
+
     return __;
   })();
 
@@ -1884,8 +1954,6 @@ async function handle_data(arg0: {
     // TWO STEP: a .ts cannot carry mov_text subtitles, they are added in step 2
     has_subs = two_step ? false : subs === false ? false : true;
 
-    console.log({ use_s3_pipe });
-
     const data = await NS_ffmpeg_v2.F_build_ffmpeg_flags_t1({
       playback: new_playback,
       subtitles: two_step || subs === false ? undefined : subtitles,
@@ -1897,8 +1965,6 @@ async function handle_data(arg0: {
       dl_to_s3: use_s3_pipe
     });
 
-    console.log({ data });
-
     return data;
   }
 

src/sub-router/stream.ts

@@ -1,11 +1,11 @@
 import { type Context, Elysia, t } from "elysia";
 import { NS_runtime, IS_HF } from "@x/types/env.runtime";
-import { offline_store } from "../lib/offline-store";
-import { redis } from "../lib/upstash";
+import { offline_store } from "@x/ff-dl/offline-store";
 import { NS_promise } from "@x/core/client/promise.ts";
 import { NS_error } from "@x/core/client/error.ts";
 import { NS_ff_dl_stream } from "@x/ff-dl/dl-stream";
 import { NS_Misc } from "@x/core/client/misc.ts";
+import { redis } from "@x/post-process/cache";
 
 const APP_ACCOUNT_ABC = () =>
   NS_runtime.ENV_VAR_DB?.APP_ACCOUNT_ABC || process.env.APP_ACCOUNT_ABC;
@@ -44,7 +44,7 @@ const sub_router = new Elysia({ prefix: "/stream" }).get(
           if (_) return _;
         }
 
-        const _ = await redis().get(data_key);
+        const _ = await redis()?.get(data_key);
 
         return _;
       } catch {