feat: adding auto refresh of download sources

src/main/entity/download-source.entity.ts

@@ -7,6 +7,7 @@ import {
   OneToMany,
 } from "typeorm";
 import { Repack } from "./repack.entity";
+import { DownloadSourceStatus } from "@shared";
 
 @Entity("download_source")
 export class DownloadSource {
@@ -19,6 +20,12 @@ export class DownloadSource {
   @Column("text")
   name: string;
 
+  @Column("text", { nullable: true })
+  etag: string | null;
+
+  @Column("text", { default: "online" })
+  status: DownloadSourceStatus;
+
   @OneToMany(() => Repack, (repack) => repack.downloadSource, { cascade: true })
   repacks: Repack[];
 

src/main/events/download-sources/add-download-source.ts

@@ -1,12 +1,11 @@
 import { registerEvent } from "../register-event";
-import { chunk } from "lodash-es";
 import { dataSource } from "@main/data-source";
-import { DownloadSource, Repack } from "@main/entity";
+import { DownloadSource } from "@main/entity";
 import axios from "axios";
-import { QueryDeepPartialEntity } from "typeorm/query-builder/QueryPartialEntity";
 import { downloadSourceSchema } from "../helpers/validators";
 import { repackRepository } from "@main/repository";
 import { repacksWorker } from "@main/workers";
+import { insertDownloadsFromSource } from "@main/helpers";
 
 const addDownloadSource = async (
   _event: Electron.IpcMainInvokeEvent,
@@ -22,30 +21,12 @@ const addDownloadSource = async (
         .getRepository(DownloadSource)
         .save({ url, name: source.name });
 
-      const repacks: QueryDeepPartialEntity<Repack>[] = source.downloads.map(
+      await insertDownloadsFromSource(
-        (download) => ({
+        transactionalEntityManager,
-          title: download.title,
+        downloadSource,
-          magnet: download.uris[0],
+        source.downloads
-          fileSize: download.fileSize,
-          repacker: source.name,
-          uploadDate: download.uploadDate,
-          downloadSource: { id: downloadSource.id },
-        })
       );
 
-      const downloadsChunks = chunk(repacks, 800);
-
-      for (const chunk of downloadsChunks) {
-        await transactionalEntityManager
-          .getRepository(Repack)
-          .createQueryBuilder()
-          .insert()
-          .values(chunk)
-          .updateEntity(false)
-          .orIgnore()
-          .execute();
-      }
-
       return downloadSource;
     }
   );

src/main/helpers/download-source.ts

@@ -0,0 +1,69 @@
+import { dataSource } from "@main/data-source";
+import { DownloadSource, Repack } from "@main/entity";
+import { downloadSourceSchema } from "@main/events/helpers/validators";
+import { downloadSourceRepository } from "@main/repository";
+import { downloadSourceWorker } from "@main/workers";
+import { chunk } from "lodash-es";
+import type { EntityManager } from "typeorm";
+import type { QueryDeepPartialEntity } from "typeorm/query-builder/QueryPartialEntity";
+import { z } from "zod";
+
+export const insertDownloadsFromSource = async (
+  trx: EntityManager,
+  downloadSource: DownloadSource,
+  downloads: z.infer<typeof downloadSourceSchema>["downloads"]
+) => {
+  const repacks: QueryDeepPartialEntity<Repack>[] = downloads.map(
+    (download) => ({
+      title: download.title,
+      magnet: download.uris[0],
+      fileSize: download.fileSize,
+      repacker: downloadSource.name,
+      uploadDate: download.uploadDate,
+      downloadSource: { id: downloadSource.id },
+    })
+  );
+
+  const downloadsChunks = chunk(repacks, 800);
+
+  for (const chunk of downloadsChunks) {
+    await trx
+      .getRepository(Repack)
+      .createQueryBuilder()
+      .insert()
+      .values(chunk)
+      .updateEntity(false)
+      .orIgnore()
+      .execute();
+  }
+};
+
+export const fetchDownloadSourcesAndUpdate = async () => {
+  const downloadSources = await downloadSourceRepository.find({
+    order: {
+      id: "desc",
+    },
+  });
+
+  const results = await downloadSourceWorker.run(downloadSources, {
+    name: "getUpdatedRepacks",
+  });
+
+  await dataSource.transaction(async (transactionalEntityManager) => {
+    for (const result of results) {
+      await transactionalEntityManager.getRepository(DownloadSource).update(
+        { id: result.id },
+        {
+          etag: result.etag,
+          status: result.status,
+        }
+      );
+
+      await insertDownloadsFromSource(
+        transactionalEntityManager,
+        result,
+        result.downloads
+      );
+    }
+  });
+};

src/main/helpers/index.ts

@@ -58,3 +58,4 @@ export const requestWebPage = async (url: string) => {
 };
 
 export * from "./ps";
+export * from "./download-source";

src/main/main.ts

@@ -8,6 +8,7 @@ import { UserPreferences } from "./entity";
 import { RealDebridClient } from "./services/real-debrid";
 import { Not } from "typeorm";
 import { repacksWorker } from "./workers";
+import { fetchDownloadSourcesAndUpdate } from "./helpers";
 
 startMainLoop();
 
@@ -27,15 +28,14 @@ const loadState = async (userPreferences: UserPreferences | null) => {
 
   if (game) DownloadManager.startDownload(game);
 
-  repackRepository
+  const repacks = await repackRepository.find({
-    .find({
+    order: {
-      order: {
+      createdAt: "DESC",
-        createdAt: "DESC",
+    },
-      },
+  });
-    })
+
-    .then((repacks) => {
+  repacksWorker.run(repacks, { name: "setRepacks" });
-      repacksWorker.run(repacks, { name: "setRepacks" });
+  fetchDownloadSourcesAndUpdate();
-    });
 };
 
 userPreferencesRepository

src/main/workers/download-source.worker.ts

@@ -0,0 +1,49 @@
+import { downloadSourceSchema } from "@main/events/helpers/validators";
+import { DownloadSourceStatus } from "@shared";
+import type { DownloadSource } from "@types";
+import axios, { AxiosError, AxiosHeaders } from "axios";
+import { z } from "zod";
+
+export type DownloadSourceResponse = z.infer<typeof downloadSourceSchema> & {
+  etag: string | null;
+  status: DownloadSourceStatus;
+};
+
+export const getUpdatedRepacks = async (downloadSources: DownloadSource[]) => {
+  const results: DownloadSourceResponse[] = [];
+
+  for (const downloadSource of downloadSources) {
+    const headers = new AxiosHeaders();
+
+    if (downloadSource.etag) {
+      headers.set("If-None-Match", downloadSource.etag);
+    }
+
+    try {
+      const response = await axios.get(downloadSource.url, {
+        headers,
+      });
+
+      const source = downloadSourceSchema.parse(response.data);
+
+      results.push({
+        ...downloadSource,
+        downloads: source.downloads,
+        etag: response.headers["etag"],
+        status: DownloadSourceStatus.UpToDate,
+      });
+    } catch (err: unknown) {
+      const isNotModified = (err as AxiosError).response?.status === 304;
+
+      results.push({
+        ...downloadSource,
+        downloads: [],
+        status: isNotModified
+          ? DownloadSourceStatus.UpToDate
+          : DownloadSourceStatus.Errored,
+      });
+    }
+  }
+
+  return results;
+};

src/main/workers/index.ts

@@ -1,6 +1,7 @@
 import path from "node:path";
 import steamGamesWorkerPath from "./steam-games.worker?modulePath";
 import repacksWorkerPath from "./repacks.worker?modulePath";
+import downloadSourceWorkerPath from "./download-source.worker?modulePath";
 
 import Piscina from "piscina";
 
@@ -16,3 +17,7 @@ export const steamGamesWorker = new Piscina({
 export const repacksWorker = new Piscina({
   filename: repacksWorkerPath,
 });
+
+export const downloadSourceWorker = new Piscina({
+  filename: downloadSourceWorkerPath,
+});

src/renderer/src/components/badge/badge.css.ts

@@ -0,0 +1,12 @@
+import { style } from "@vanilla-extract/css";
+import { SPACING_UNIT } from "../../theme.css";
+
+export const badge = style({
+  color: "#c0c1c7",
+  fontSize: "10px",
+  padding: `${SPACING_UNIT / 2}px ${SPACING_UNIT}px`,
+  border: "solid 1px #c0c1c7",
+  borderRadius: "4px",
+  display: "flex",
+  alignItems: "center",
+});

src/renderer/src/components/badge/badge.tsx

@@ -0,0 +1,14 @@
+import React from "react";
+import * as styles from "./badge.css";
+
+export interface BadgeProps {
+  children: React.ReactNode;
+}
+
+export function Badge({ children }: BadgeProps) {
+  return (
+    <div className={styles.badge}>
+      <span>{children}</span>
+    </div>
+  );
+}

src/renderer/src/components/game-card/game-card.css.ts

@@ -69,16 +69,7 @@ export const downloadOptions = style({
   padding: "0",
   gap: `${SPACING_UNIT}px`,
   flexWrap: "wrap",
-});
+  listStyle: "none",
-
-export const downloadOption = style({
-  color: "#c0c1c7",
-  fontSize: "10px",
-  padding: `${SPACING_UNIT / 2}px ${SPACING_UNIT}px`,
-  border: "solid 1px #c0c1c7",
-  borderRadius: "4px",
-  display: "flex",
-  alignItems: "center",
 });
 
 export const specifics = style({

src/renderer/src/components/game-card/game-card.tsx

@@ -5,6 +5,7 @@ import SteamLogo from "@renderer/assets/steam-logo.svg?react";
 
 import * as styles from "./game-card.css";
 import { useTranslation } from "react-i18next";
+import { Badge } from "../badge/badge";
 
 export interface GameCardProps
   extends React.DetailedHTMLProps<
@@ -39,8 +40,8 @@ export function GameCard({ game, ...props }: GameCardProps) {
           {uniqueRepackers.length > 0 ? (
             <ul className={styles.downloadOptions}>
               {uniqueRepackers.map((repacker) => (
-                <li key={repacker} className={styles.downloadOption}>
+                <li key={repacker}>
-                  <span>{repacker}</span>
+                  <Badge>{repacker}</Badge>
                 </li>
               ))}
             </ul>

src/renderer/src/components/index.ts

@@ -10,3 +10,4 @@ export * from "./checkbox-field/checkbox-field";
 export * from "./link/link";
 export * from "./select-field/select-field";
 export * from "./toast/toast";
+export * from "./badge/badge";

src/renderer/src/pages/downloads/downloads.css.ts

@@ -21,16 +21,6 @@ export const downloadTitle = style({
   },
 });
 
-export const downloaderName = style({
-  color: "#c0c1c7",
-  fontSize: "10px",
-  padding: `${SPACING_UNIT / 2}px ${SPACING_UNIT}px`,
-  border: "solid 1px #c0c1c7",
-  borderRadius: "4px",
-  display: "flex",
-  alignItems: "center",
-});
-
 export const downloads = style({
   width: "100%",
   gap: `${SPACING_UNIT * 2}px`,

src/renderer/src/pages/downloads/downloads.tsx

@@ -1,7 +1,7 @@
 import { useTranslation } from "react-i18next";
 import { useNavigate } from "react-router-dom";
 
-import { Button, TextField } from "@renderer/components";
+import { Badge, Button, TextField } from "@renderer/components";
 import {
   buildGameDetailsPath,
   formatDownloadProgress,
@@ -255,9 +255,7 @@ export function Downloads() {
                   />
 
                   <div className={styles.downloadCoverContent}>
-                    <small className={styles.downloaderName}>
+                    <Badge>{DOWNLOADER_NAME[game.downloader]}</Badge>
-                      {DOWNLOADER_NAME[game.downloader]}
-                    </small>
                   </div>
                 </div>
               </div>

src/renderer/src/pages/settings/add-download-source-modal.tsx

@@ -75,6 +75,7 @@ export function AddDownloadSourceModal({
 
           <Button
             type="button"
+            theme="outline"
             style={{ alignSelf: "flex-end" }}
             onClick={handleValidateDownloadSource}
             disabled={isLoading || !value}

src/renderer/src/pages/settings/settings-download-sources.css.ts

@@ -6,6 +6,13 @@ export const downloadSourceField = style({
   gap: `${SPACING_UNIT}px`,
 });
 
+export const downloadSources = style({
+  padding: "0",
+  gap: `${SPACING_UNIT * 2}px`,
+  display: "flex",
+  flexDirection: "column",
+});
+
 export const downloadSourceItem = style({
   display: "flex",
   flexDirection: "column",
@@ -20,5 +27,11 @@ export const downloadSourceItemHeader = style({
   marginBottom: `${SPACING_UNIT}px`,
   display: "flex",
   flexDirection: "column",
-  gap: `${SPACING_UNIT / 2}px`,
+  gap: `${SPACING_UNIT}px`,
+});
+
+export const downloadSourceItemTitle = style({
+  display: "flex",
+  gap: `${SPACING_UNIT}px`,
+  alignItems: "center",
 });

src/renderer/src/pages/settings/settings-download-sources.tsx

@@ -1,6 +1,6 @@
 import { useEffect, useState } from "react";
 
-import { TextField, Button } from "@renderer/components";
+import { TextField, Button, Badge } from "@renderer/components";
 import { useTranslation } from "react-i18next";
 
 import * as styles from "./settings-download-sources.css";
@@ -62,38 +62,46 @@ export function SettingsDownloadSources() {
         {t("add_download_source")}
       </Button>
 
-      {downloadSources.map((downloadSource) => (
+      <ul className={styles.downloadSources}>
-        <div key={downloadSource.id} className={styles.downloadSourceItem}>
+        {downloadSources.map((downloadSource) => (
-          <div className={styles.downloadSourceItemHeader}>
+          <li key={downloadSource.id} className={styles.downloadSourceItem}>
-            <h3>{downloadSource.name}</h3>
+            <div className={styles.downloadSourceItemHeader}>
-            <small>
+              <div className={styles.downloadSourceItemTitle}>
-              {t("download_options", {
+                <h2>{downloadSource.name}</h2>
-                count: downloadSource.repackCount,
-                countFormatted: downloadSource.repackCount.toLocaleString(),
-              })}
-            </small>
-          </div>
 
-          <div className={styles.downloadSourceField}>
+                <Badge>{downloadSource.status}</Badge>
-            <TextField
+              </div>
-              label={t("download_source_url")}
-              value={downloadSource.url}
-              readOnly
-              disabled
-            />
 
-            <Button
+              <small>
-              type="button"
+                {t("download_options", {
-              theme="outline"
+                  count: downloadSource.repackCount,
-              style={{ alignSelf: "flex-end" }}
+                  countFormatted: downloadSource.repackCount.toLocaleString(),
-              onClick={() => handleRemoveSource(downloadSource.id)}
+                })}
-            >
+              </small>
-              <NoEntryIcon />
+            </div>
-              {t("remove_download_source")}
+
-            </Button>
+            <div className={styles.downloadSourceField}>
-          </div>
+              <TextField
-        </div>
+                label={t("download_source_url")}
-      ))}
+                value={downloadSource.url}
+                readOnly
+                theme="dark"
+                disabled
+              />
+
+              <Button
+                type="button"
+                theme="outline"
+                style={{ alignSelf: "flex-end" }}
+                onClick={() => handleRemoveSource(downloadSource.id)}
+              >
+                <NoEntryIcon />
+                {t("remove_download_source")}
+              </Button>
+            </div>
+          </li>
+        ))}
+      </ul>
     </>
   );
 }

src/shared/index.ts

@@ -3,6 +3,11 @@ export enum Downloader {
   Torrent,
 }
 
+export enum DownloadSourceStatus {
+  UpToDate,
+  Errored,
+}
+
 const FORMAT = ["B", "KB", "MB", "GB", "TB", "PB", "EB", "ZB", "YB"];
 
 export const formatBytes = (bytes: number): string => {

src/types/index.ts

@@ -1,5 +1,5 @@
 import type { Aria2Status } from "aria2";
-import type { Downloader } from "@shared";
+import type { DownloadSourceStatus, Downloader } from "@shared";
 import { ProgressInfo, UpdateInfo } from "electron-updater";
 
 export type GameShop = "steam" | "epic";
@@ -240,6 +240,8 @@ export interface DownloadSource {
   name: string;
   url: string;
   repackCount: number;
+  status: DownloadSourceStatus;
+  etag: string | null;
   createdAt: Date;
   updatedAt: Date;
 }