Create scrape backend. Create fishes endpoint

scraper/nodemon.json

@@ -0,0 +1,6 @@
+{
+  "watch": ["src"],
+  "ext": "ts",
+  "ignore": ["src/**/*.spec.ts"],
+  "exec": "ts-node ./src/server.ts"
+}

scraper/package.json

@@ -0,0 +1,26 @@
+{
+  "name": "scraper",
+  "version": "1.0.0",
+  "description": "Scraper app to get data from animal crossing wiki",
+  "main": "index.js",
+  "scripts": {
+    "test": "echo \"Error: no test specified\" && exit 1",
+    "start": "nodemon"
+  },
+  "author": "Tatiana Inama",
+  "license": "ISC",
+  "dependencies": {
+    "@types/cheerio": "^0.22.17",
+    "@types/express": "^4.17.6",
+    "@types/node-fetch": "^2.5.6",
+    "@types/ramda": "^0.27.3",
+    "@types/request-promise": "^4.1.46",
+    "axios": "^0.19.2",
+    "cheerio": "^1.0.0-rc.3",
+    "express": "^4.17.1",
+    "nodemon": "^2.0.3",
+    "ramda": "^0.27.0",
+    "ts-node": "^8.8.2",
+    "typescript": "^3.8.3"
+  }
+}

scraper/src/scraper.ts

@@ -0,0 +1,77 @@
+import cheerio from 'cheerio';
+import axios from 'axios';
+import { splitAt } from 'ramda';
+
+type DataParser = (td: Cheerio) => ({
+  [x: string]: number | string | [number, number][]
+});
+
+const mkLocation = (location: string): string => {
+  switch (location) {
+    case 'River (Clifftop)' || 'River (Clifftop)  Pond':
+      return 'River (Clifftop)';
+    default:
+      return location;
+  }
+}
+
+const mkTime = (time: string): [number, number][] => {
+  switch (time) {
+    case '9 AM - 4 PM':
+      return [[9, 16]];
+    case '4 PM - 9 AM':
+      return [[16, 9]];
+    case '9 PM - 4 AM':
+      return [[21, 4]];
+    case '9 AM - 4 PM & 9 PM - 4 AM':
+      return [[9, 16], [21, 4]];
+    case '4 AM - 9 PM':
+      return [[4, 21]];
+    default:
+      return [[0, 24]];
+  }
+}
+
+const fishPropMap: DataParser[] = [
+  td => ({name: td.text().trim()}),
+  td => ({ img: td.find('a').attr('href') || '' }),
+  td => ({ price: parseInt(td.text().trim())}),
+  td => ({ location: mkLocation(td.text().trim())}),
+  td => ({ shadowSize: parseInt(td.text().trim())}),
+  td => ({ time: mkTime(td.text().trim())})
+]
+
+const parseMonths = (tr: CheerioElement[]) => {
+  return tr.reduce<number[]>((calendar, td, month) => {
+    return td.firstChild.data?.trim() === '✓' ? [ ...calendar, month+1 ] : [ ...calendar ]
+  }, [])
+}
+
+const scrape = async () => {
+  const html = await axios.get('https://animalcrossing.fandom.com/wiki/Fish_(New_Horizons)');
+  const $ = cheerio.load(html.data, { normalizeWhitespace: true });
+  
+  const rows = $('[title="Northern Hemisphere"] table.roundy.sortable tbody tr').toArray().slice(1);
+  const fishes = rows.reduce((fishes, rowData) => {
+    const [ fishData, months ] = splitAt(6, rowData.children.slice(1));
+
+    const fish = {
+      ...fishData.reduce(( fish, td, i ) => {
+        return {
+          ...fish,
+          ...fishPropMap[i]($(td))
+        };
+      }, {}),
+      months: parseMonths(months)
+    };
+
+    return [
+      ...fishes,
+      fish
+    ]
+  }, [] as object[]);
+
+  return fishes;
+}
+
+export default scrape;

scraper/src/server.ts

@@ -0,0 +1,22 @@
+import express from 'express';
+
+import scrape from './scraper';
+
+const app: express.Application = express();
+const port: number = 3001;
+
+app.get('/', (req, res) => {
+  scrape();
+  res.send('Hello World!')
+});
+
+app.get('/fishes', (req, res) => {
+  return scrape().then(fishes => {
+    return res.send(fishes);
+  })
+})
+
+app.listen(port, () => {
+  console.log(`Scrapper app listening at http://localhost:${port}`);
+  scrape();
+});

scraper/tsconfig.json

@@ -0,0 +1,66 @@
+{
+  "compilerOptions": {
+    /* Basic Options */
+    // "incremental": true,                   /* Enable incremental compilation */
+    "target": "es5",                          /* Specify ECMAScript target version: 'ES3' (default), 'ES5', 'ES2015', 'ES2016', 'ES2017', 'ES2018', 'ES2019', 'ES2020', or 'ESNEXT'. */
+    "module": "commonjs",                     /* Specify module code generation: 'none', 'commonjs', 'amd', 'system', 'umd', 'es2015', 'es2020', or 'ESNext'. */
+    // "lib": [],                             /* Specify library files to be included in the compilation. */
+    // "allowJs": true,                       /* Allow javascript files to be compiled. */
+    // "checkJs": true,                       /* Report errors in .js files. */
+    // "jsx": "preserve",                     /* Specify JSX code generation: 'preserve', 'react-native', or 'react'. */
+    // "declaration": true,                   /* Generates corresponding '.d.ts' file. */
+    // "declarationMap": true,                /* Generates a sourcemap for each corresponding '.d.ts' file. */
+    // "sourceMap": true,                     /* Generates corresponding '.map' file. */
+    // "outFile": "./",                       /* Concatenate and emit output to single file. */
+    // "outDir": "./",                        /* Redirect output structure to the directory. */
+    // "rootDir": "./",                       /* Specify the root directory of input files. Use to control the output directory structure with --outDir. */
+    // "composite": true,                     /* Enable project compilation */
+    // "tsBuildInfoFile": "./",               /* Specify file to store incremental compilation information */
+    // "removeComments": true,                /* Do not emit comments to output. */
+    // "noEmit": true,                        /* Do not emit outputs. */
+    // "importHelpers": true,                 /* Import emit helpers from 'tslib'. */
+    // "downlevelIteration": true,            /* Provide full support for iterables in 'for-of', spread, and destructuring when targeting 'ES5' or 'ES3'. */
+    // "isolatedModules": true,               /* Transpile each file as a separate module (similar to 'ts.transpileModule'). */
+
+    /* Strict Type-Checking Options */
+    "strict": true,                           /* Enable all strict type-checking options. */
+    // "noImplicitAny": true,                 /* Raise error on expressions and declarations with an implied 'any' type. */
+    // "strictNullChecks": true,              /* Enable strict null checks. */
+    // "strictFunctionTypes": true,           /* Enable strict checking of function types. */
+    // "strictBindCallApply": true,           /* Enable strict 'bind', 'call', and 'apply' methods on functions. */
+    // "strictPropertyInitialization": true,  /* Enable strict checking of property initialization in classes. */
+    // "noImplicitThis": true,                /* Raise error on 'this' expressions with an implied 'any' type. */
+    // "alwaysStrict": true,                  /* Parse in strict mode and emit "use strict" for each source file. */
+
+    /* Additional Checks */
+    // "noUnusedLocals": true,                /* Report errors on unused locals. */
+    // "noUnusedParameters": true,            /* Report errors on unused parameters. */
+    // "noImplicitReturns": true,             /* Report error when not all code paths in function return a value. */
+    // "noFallthroughCasesInSwitch": true,    /* Report errors for fallthrough cases in switch statement. */
+
+    /* Module Resolution Options */
+    // "moduleResolution": "node",            /* Specify module resolution strategy: 'node' (Node.js) or 'classic' (TypeScript pre-1.6). */
+    // "baseUrl": "./",                       /* Base directory to resolve non-absolute module names. */
+    // "paths": {},                           /* A series of entries which re-map imports to lookup locations relative to the 'baseUrl'. */
+    // "rootDirs": [],                        /* List of root folders whose combined content represents the structure of the project at runtime. */
+    // "typeRoots": [],                       /* List of folders to include type definitions from. */
+    // "types": [],                           /* Type declaration files to be included in compilation. */
+    // "allowSyntheticDefaultImports": true,  /* Allow default imports from modules with no default export. This does not affect code emit, just typechecking. */
+    "esModuleInterop": true,                  /* Enables emit interoperability between CommonJS and ES Modules via creation of namespace objects for all imports. Implies 'allowSyntheticDefaultImports'. */
+    // "preserveSymlinks": true,              /* Do not resolve the real path of symlinks. */
+    // "allowUmdGlobalAccess": true,          /* Allow accessing UMD globals from modules. */
+
+    /* Source Map Options */
+    // "sourceRoot": "",                      /* Specify the location where debugger should locate TypeScript files instead of source locations. */
+    // "mapRoot": "",                         /* Specify the location where debugger should locate map files instead of generated locations. */
+    // "inlineSourceMap": true,               /* Emit a single file with source maps instead of having a separate file. */
+    // "inlineSources": true,                 /* Emit the source alongside the sourcemaps within a single file; requires '--inlineSourceMap' or '--sourceMap' to be set. */
+
+    /* Experimental Options */
+    // "experimentalDecorators": true,        /* Enables experimental support for ES7 decorators. */
+    // "emitDecoratorMetadata": true,         /* Enables experimental support for emitting type metadata for decorators. */
+
+    /* Advanced Options */
+    "forceConsistentCasingInFileNames": true  /* Disallow inconsistently-cased references to the same file. */
+  }
+}