feat: Better Web GUI with bot controls (#3250)

Co-authored-by: copilot-swe-agent[bot] <198982749+Copilot@users.noreply.github.com>

Author: Mr-Tech-13

README.md

@@ -14,6 +14,7 @@ First and foremost, this service _will not_ automatically buy for you.
 
 - **Checks stock continuously** -- runs 24/7, 365, looking for the items you want.
 - **Ready for checkout** -- ability to add to cart when available and even opens the browser for you.
+- **Live dashboard** -- optional web interface with a matrix view of selected stores and series, filter controls, dotenv editing, and restart control.
 - **Notifications galore** -- when you're not by your computer, worry free with notifications to most platforms and devices when an item comes in stock.
 
 ## Quick start
@@ -25,4 +26,6 @@ git clone https://github.com/jef/streetmerchant.git
 cd streetmerchant && npm i && npm run start
 ```
 
+To enable the web dashboard, set `WEB_PORT` in `dotenv` and open `http://localhost:<WEB_PORT>` while streetmerchant is running.
+
 For more information and customization, visit [jef.buzz/streetmerchant/getting-started](https://jef.buzz/streetmerchant/getting-started).

docs/getting-started.md

@@ -25,6 +25,26 @@ You do not need any computer skills, smarts, or anything of that nature. You are
 
 At any point you want the program to stop, use ++ctrl+c++.
 
+## Using the web dashboard
+
+If you want to manage streetmerchant from the browser while it is running, set `WEB_PORT` in your `dotenv` file.
+
+```shell
+WEB_PORT=8080
+```
+
+After you start the app, open `http://localhost:8080`.
+
+The dashboard includes:
+
+- A live matrix with selected stores across the top and selected series on the left.
+- Store, series, and model menus that update the running configuration.
+- A settings editor for the active `dotenv` file.
+- A restart button to reload the bot without closing the dashboard.
+
+???+ note
+    Filter changes made in the dashboard are persisted back to the active `dotenv` file. Some settings still require a restart before they fully take effect.
+
 ???+ tip
     Community based help can also be found on the [wiki](https://github.com/jef/streetmerchant/wiki). Feel free to check that out if you're having problems running. If you're still having problems running, you're probably not the first. Make some searches through the [GitHub issues](https://github.com/jef/streetmerchant/issues) before making one.
 

docs/index.md

@@ -6,6 +6,7 @@ First and foremost, this service _will not_ automatically buy for you.
 
 - **Checks stock continuously** -- runs 24/7, 365, looking for the items you want.
 - **Ready for checkout** -- ability to add to cart when available and even opens the browser for you.
+- **Live dashboard** -- optional web interface with a matrix view of selected stores and series, filter controls, dotenv editing, and restart control.
 - **Notifications galore** -- when you're not by your computer, worry free with notifications to most platforms and devices when an item comes in stock.
 
 ## Getting started

docs/reference/application.md

@@ -7,6 +7,7 @@
 | `HEADLESS` | Puppeteer to run headless or not. Debugging related, default: `true` |
 | `INCOGNITO` | Puppeteer to run incognito or not. Debugging related, default: `false` |
 | `IN_STOCK_WAIT_TIME` | Time to wait between requests to the same link if it has that card in stock. In seconds, default: `0` |
+| `LOOKUP_THREADS` | Maximum number of product links checked concurrently per store loop, default: `1` |
 | `LOG_LEVEL` | [Logging levels](https://github.com/winstonjs/winston#logging-levels). Debugging related, default: `info` |
 | `LOW_BANDWIDTH` | Blocks images/fonts to reduce traffic. Disables ad blocker, default: `false` |
 | `NVIDIA_ADD_TO_CART_ATTEMPTS` | Maximum number of attempts add an item to card in the Nvidia storefront, default: `10` |
@@ -20,15 +21,17 @@
 | `PROXY_PROTOCOL` | Protocol of proxy server, such as `socks5`, default: `http` |
 | `PROXY_ADDRESS` | IP Address or fqdn of proxy server |
 | `PROXY_PORT` | TCP Port number on which the proxy is listening for connections, default: `80` |
+| `RANDOMIZE_LOOKUP_ORDER` | Shuffle store and product lookup order on each pass, default: `false` |
 | `RESTART_TIME` | Restarts chrome after defined milliseconds. `0` for never, default: `0` |
 | `SCREENSHOT` | Capture screenshot of page if a card is found, default: `true` |
 | `SCREENSHOT_DIR` | The directory for saving the screenshots, default: `screenshots` |
 | `USER_AGENT` | Custom user agent used for requests |
-| `WEB_PORT` | Starts a webserver to be able to control the bot while it is running. Setting this value starts this service. |
+| `WEB_PORT` | Starts a webserver to control the bot while it is running. The dashboard includes a live matrix view, filter menus, dotenv editing, and a restart control. |
 
 ???+ info
     There is more information on proxy settings in the [Proxy documentation](proxy.md).
 
 ???+ tip
     - You can also have a list of proxies that are rotated while searching stores. Proxies can be read from a file named `STORENAME.proxies` in the format of `socks5://username:password@ip`; one per line.
     - Data usage is [known to be high](https://github.com/jef/streetmerchant/issues?q=is%3Aissue+sort%3Aupdated-desc+bandwidth). This is expected as the program scrapes many websites in parallel 24/7. To help reduce this, use `LOW_BANDWIDTH="true"`. We are looking into other solutions as well, but is low priority.
+    - Increasing `LOOKUP_THREADS` can improve throughput, but it also increases request pressure and rate-limit risk.

docs/reference/filter.md

@@ -54,6 +54,9 @@
 ???+ note
     For `MAX_PRICE_SERIES_*` variables: Use whole numbers only (no currency symbol is required). Avoid using any commas or decimal points. Example: `1234`. Merchandise found above this price will be skipped.
 
+???+ info
+    When `WEB_PORT` is enabled, the web dashboard can update `STORES`, `SHOW_ONLY_SERIES`, and `SHOW_ONLY_MODELS` from the browser. Those changes are also written back to the active `dotenv` file.
+
 ## Supported stores
 
 Used with the `STORES` variable.
@@ -122,7 +125,7 @@ Used with the `STORES` variable.
 | Drako | IT | `drako` |
 | DustinHome | NO | `dustinhome-no` |
 | eBuyer | UK | `ebuyer` |
-| El Corte Inglés | ES | `elcorteingles` |
+| El Corte Ingles | ES | `elcorteingles` |
 | Eletronicamente | ES | `eletronicamente` |
 | Elkjop | NO | `elkjop` |
 | ePrice | IT | `eprice` |

dotenv-example

@@ -74,6 +74,8 @@ IN_STOCK_WAIT_TIME=
 INCOGNITO=
 LOG_LEVEL=
 LOW_BANDWIDTH=
+LOOKUP_THREADS=
+RANDOMIZE_LOOKUP_ORDER=
 MAX_PRICE_SERIES_3060=
 MAX_PRICE_SERIES_3060TI=
 MAX_PRICE_SERIES_3070=

src/config.ts

@@ -3,24 +3,36 @@ import {existsSync, readFileSync} from 'fs';
 import path from 'path';
 import {banner} from './banner';
 
-if (process.env.npm_config_conf) {
-  if (
-    existsSync(path.resolve(__dirname, '../../' + process.env.npm_config_conf))
-  ) {
-    dotenv.config({
-      path: path.resolve(__dirname, '../../' + process.env.npm_config_conf),
-    });
-  } else {
-    dotenv.config({path: path.resolve(__dirname, '../../.env')});
+export function getActiveConfigPath() {
+  if (process.env.npm_config_conf) {
+    const configuredPath = path.resolve(
+      __dirname,
+      '../../' + process.env.npm_config_conf
+    );
+    if (existsSync(configuredPath)) {
+      return configuredPath;
+    }
+
+    return path.resolve(__dirname, '../../.env');
   }
-} else if (existsSync(path.resolve(__dirname, '../../dotenv'))) {
-  dotenv.config({path: path.resolve(__dirname, '../../dotenv')});
-} else if (existsSync(path.resolve(__dirname, '../dotenv'))) {
-  dotenv.config({path: path.resolve(__dirname, '../dotenv')});
-} else {
-  dotenv.config({path: path.resolve(__dirname, '../../.env')});
+
+  const rootDotenv = path.resolve(__dirname, '../../dotenv');
+  if (existsSync(rootDotenv)) {
+    return rootDotenv;
+  }
+
+  const buildDotenv = path.resolve(__dirname, '../dotenv');
+  if (existsSync(buildDotenv)) {
+    return buildDotenv;
+  }
+
+  return path.resolve(__dirname, '../../.env');
 }
 
+export const activeConfigPath = getActiveConfigPath();
+
+dotenv.config({path: activeConfigPath});
+
 console.info(
   banner.render(
     envOrBoolean(process.env.ASCII_BANNER, false),
@@ -414,6 +426,8 @@ const nvidia = {
 const page = {
   height: 1080,
   inStockWaitTime: envOrNumber(process.env.IN_STOCK_WAIT_TIME),
+  lookupThreads: envOrNumber(process.env.LOOKUP_THREADS, 1),
+  randomizeLookupOrder: envOrBoolean(process.env.RANDOMIZE_LOOKUP_ORDER, false),
   screenshot: envOrBoolean(process.env.SCREENSHOT),
   screenshotDir: envOrString(process.env.SCREENSHOT_DIR, 'screenshots'),
   timeout: envOrNumber(process.env.PAGE_TIMEOUT, 30000),

src/index.ts

@@ -6,31 +6,46 @@ import {getSleepTime} from './util';
 import {logger} from './logger';
 import {storeList} from './store/model';
 import {tryLookupAndLoop} from './store';
+import {setRestartBotHandler} from './runtime-control';
 
 let browser: Browser | undefined;
+let runGeneration = 0;
+
+function shuffleArray<T>(items: T[]): T[] {
+  const shuffled = [...items];
+  for (let i = shuffled.length - 1; i > 0; i--) {
+    const j = Math.floor(Math.random() * (i + 1));
+    [shuffled[i], shuffled[j]] = [shuffled[j], shuffled[i]];
+  }
+
+  return shuffled;
+}
 
 async function sleep(ms: number) {
   return new Promise(resolve => setTimeout(resolve, ms));
 }
 
-/**
- * Schedules a restart of the bot
- */
-async function restartMain() {
+async function scheduleRestart(generation: number) {
   if (config.restartTime > 0) {
     await sleep(config.restartTime);
-    await stop();
-    loopMain();
+    if (generation !== runGeneration) {
+      return;
+    }
+
+    await restartBot();
   }
 }
 
-/**
- * Starts the bot.
- */
-async function main() {
+async function startBot(startApiServer: boolean) {
+  const generation = ++runGeneration;
   browser = await launchBrowser();
+  void scheduleRestart(generation);
+
+  const stores = config.page.randomizeLookupOrder
+    ? shuffleArray([...storeList.values()])
+    : [...storeList.values()];
 
-  for (const store of storeList.values()) {
+  for (const store of stores) {
     logger.debug('store links', {meta: {links: store.links}});
     if (store.setupAction !== undefined) {
       store.setupAction(browser);
@@ -39,35 +54,43 @@ async function main() {
     setTimeout(tryLookupAndLoop, getSleepTime(store), browser, store);
   }
 
-  await startAPIServer();
+  if (startApiServer) {
+    await startAPIServer();
+  }
 }
 
-async function stop() {
-  await stopAPIServer();
+async function stopBot() {
+  runGeneration++;
 
   if (browser) {
-    // Use temporary swap variable to avoid any race condition
     const browserTemporary = browser;
     browser = undefined;
     await browserTemporary.close();
   }
 }
 
+async function stop() {
+  await stopAPIServer();
+  await stopBot();
+}
+
+export async function restartBot() {
+  logger.info('Restarting streetmerchant bot');
+  await stopBot();
+  await startBot(false);
+}
+
 async function stopAndExit() {
   await stop();
   Process.exit(0);
 }
 
-/**
- * Will continually run until user interferes.
- */
 async function loopMain() {
   try {
-    restartMain();
-    await main();
+    await startBot(true);
   } catch (error: unknown) {
     logger.error(
-      '✖ something bad happened, resetting streetmerchant in 5 seconds',
+      '✖ something bad happened, resetting streetmerchant in 5 seconds',
       error
     );
     setTimeout(loopMain, 5000);
@@ -77,15 +100,11 @@ async function loopMain() {
 export async function launchBrowser(): Promise<Browser> {
   const args: string[] = [];
 
-  // Skip Chromium Linux Sandbox
-  // https://github.com/puppeteer/puppeteer/blob/main/docs/troubleshooting.md#setting-up-chrome-linux-sandbox
   if (config.browser.isTrusted) {
     args.push('--no-sandbox');
     args.push('--disable-setuid-sandbox');
   }
 
-  // https://github.com/puppeteer/puppeteer/blob/main/docs/troubleshooting.md#tips
-  // https://stackoverflow.com/questions/48230901/docker-alpine-with-node-js-and-chromium-headless-puppeter-failed-to-launch-c
   if (config.docker) {
     args.push('--disable-dev-shm-usage');
     args.push('--no-sandbox');
@@ -95,19 +114,18 @@ export async function launchBrowser(): Promise<Browser> {
     config.browser.open = false;
   }
 
-  // Add the address of the proxy server if defined
   if (config.proxy.address) {
     args.push(
       `--proxy-server=${config.proxy.protocol}://${config.proxy.address}:${config.proxy.port}`
     );
   }
 
   if (args.length > 0) {
-    logger.info('ℹ puppeteer config: ', args);
+    logger.info('ℹ puppeteer config: ', args);
   }
 
-  await stop();
-  const browser = await Puppeteer.launch({
+  await stopBot();
+  const launchedBrowser = await Puppeteer.launch({
     args,
     defaultViewport: {
       height: config.page.height,
@@ -116,11 +134,13 @@ export async function launchBrowser(): Promise<Browser> {
     headless: config.browser.isHeadless,
   });
 
-  config.browser.userAgent = await browser.userAgent();
+  config.browser.userAgent = await launchedBrowser.userAgent();
 
-  return browser;
+  return launchedBrowser;
 }
 
+setRestartBotHandler(restartBot);
+
 void loopMain();
 
 process.on('SIGINT', stopAndExit);

src/runtime-control.ts

@@ -0,0 +1,13 @@
+let restartBotHandler: (() => Promise<void>) | undefined;
+
+export function setRestartBotHandler(handler: () => Promise<void>) {
+  restartBotHandler = handler;
+}
+
+export async function restartBot() {
+  if (!restartBotHandler) {
+    throw new Error('Restart handler has not been initialized');
+  }
+
+  await restartBotHandler();
+}