Refactor: Remove all LitK scraping logic to its owned file. Create ScrapeService. TG-1 TG-2.

ktchn/src/recipes/scrape.ts

@@ -1,127 +0,0 @@
-import RequestPromise from "request-promise";
-import Cheerio from "cheerio";
-import { Recipe, RecipeDetails, Ingredient, ComposedIngredients, Author } from './index';
-import { parse } from "recipe-ingredient-parser";
-
-type RegexMutator = (s: RegExpMatchArray) => number;
-
-const matchInt = (data: string, reg: RegExp, logic?: RegexMutator): number => {
-  let result = data.match(reg);
-  return logic ? 
-    (result ? logic(result) : 0) :
-    (result ? parseInt(result[0]) : 0)
-}
-
-const parseTime = (data: string) => {
-  let parsedHoursAsMin = matchInt(data, /(\d+) hours?/, result => parseInt(result[0])*60);
-  let parsedMinutes = matchInt(data, /(\d+) minutes?/);
-
-  return parsedHoursAsMin + parsedMinutes;
-};
-
-const LAURA_DATA: Author = {
-  name: 'Laura in the Kitchen',
-  website: 'http://www.laurainthekitchen.com',
-};
-
-const LAURA_MAP: {
-  [index:string] : {
-    key: string,
-    transform: (data: string) => number,
-  }
-} = {
-  preparationTime: {
-    key: 'Preparation',
-    transform: parseTime,
-  },
-  cookingTime: {
-    key: 'Cook time',
-    transform: parseTime,
-  },
-  servings: {
-    key: 'Servings',
-    transform: (data: string) => matchInt(data, /\d+/),
-  }
-};
-
-function parseIngredients(rawIngredient: string): Ingredient {
-  let parsed = parse(rawIngredient);
-  return {
-    name: parsed.ingredient,
-    quantity: Number(parsed.quantity) || 0,
-    unit: parsed.unit || '',
-    _original: rawIngredient,
-  };
-}
-
-function getIngredients($:CheerioSelector): ComposedIngredients[] {
-  const ingredientsScrape = $('.cs-ingredients-check-list > ul').children();
-  const isNewIngredientList = (tagName: string): boolean => tagName === 'span';
-  
-  let ingredients: ComposedIngredients[] = [];
-
-  return ingredientsScrape.toArray().reduce((list, rawIngredient)=>{
-    let last = list.length - 1;
-    let text = $(rawIngredient).text();
-    if (isNewIngredientList(rawIngredient.tagName)) {
-      return list.concat([{name: text, ingredients: []}]);
-    } else {
-      list[last].ingredients = list[last].ingredients.concat([parseIngredients(text)]);
-      return list;
-    }
-  }, ingredients);
-}
-
-function getRecipeDetails($:CheerioSelector):RecipeDetails {
-  let details = new RecipeDetails();
-  const detailsKey = Object.keys(details);
-  const data = $('.cs-recipe-details').find('div');
-  let x: any = {};
-  data.each((i, el) => {
-    let key: string = $(el).find('span').text();
-    x[key] = $(el).contents().last().text();
-  });
-
-  return detailsKey.reduce((recipeDetails, key) => {
-    let _key = LAURA_MAP[key];
-    return {
-      ...recipeDetails,
-      [key]: _key.transform(x[_key.key])
-    }
-  }, details);
-}
-
-function getRecipeName($:CheerioSelector): string {
-  return $('.cs-page-title>h1').text().trim();
-}
-
-function getInstructions($:CheerioSelector): string[] {
-  const data = $('#recipe-process').find('ul').text();
-  return data.split('\n').filter(s => s !== '').map(s => s.trim().replace(/^\d\)\s*/, ''));
-}
-
-function lauraInTheKitchen($:CheerioSelector):Recipe {
-  let recipe = new Recipe(
-    getRecipeName($),
-    LAURA_DATA,
-    getRecipeDetails($),
-    getIngredients($),
-    getInstructions($),
-  );
-  return recipe;
-}
-
-function scrape(url:string) {
-  const options = {
-    uri: url,
-    transform: (body: any) => Cheerio.load(body, {
-      normalizeWhitespace: true
-    }),
-  };
-  return RequestPromise(options).then(($) => {
-    return lauraInTheKitchen($);
-  });
-
-}
-
-export default scrape;

ktchn/src/recipes/scrape/index.ts

@@ -0,0 +1,26 @@
+import RequestPromise from "request-promise";
+import Cheerio from "cheerio";
+import Sources from './sources';
+import { ScrapingSource } from './sources/index';
+
+function selectSource(url: string): ScrapingSource|void {
+  return Sources.find((source) => {
+    return url.search(source.domain) > 0;
+  });
+}
+
+function scrape(url:string) {
+  const options = {
+    uri: url,
+    transform: (body: any) => Cheerio.load(body, {
+      normalizeWhitespace: true
+    }),
+  };
+  return RequestPromise(options).then(($) => {
+    let source = selectSource(url);
+    return source ? source.scrapeRecipe($) : new Error('source not found');
+  });
+
+}
+
+export default scrape;

ktchn/src/recipes/scrape/sources/index.ts

@@ -0,0 +1,15 @@
+import LITK from "./laura-in-the-kitchen";
+import { Recipe } from '../../index';
+
+export interface ScrapingSource {
+  name: string,
+  domain: string,
+  website?: string,
+  scrapeRecipe($: CheerioSelector): Recipe,
+}
+
+const Sources: ScrapingSource[] = [
+  LITK,
+];
+
+export default Sources;

ktchn/src/recipes/scrape/sources/laura-in-the-kitchen.ts

@@ -0,0 +1,120 @@
+import { Recipe, RecipeDetails, Ingredient, ComposedIngredients, Author } from '../../index';
+import { parse as parseIngredient } from "recipe-ingredient-parser";
+
+const LITK_AUTHOR_DATA: Author = {
+  name: 'Laura Vitale',
+  website: 'http://www.laurainthekitchen.com',
+};
+
+const SELECTORS = {
+  TITLE: '.cs-page-title > h1',
+  DETAILS: '.cs-recipe-details > div',
+  INGREDIENTS: '.cs-ingredients-check-list > ul',
+  INSTRUCTIONS: '#recipe-process > ul',
+}
+
+function getRecipeName($: CheerioSelector): string {
+  return $(SELECTORS.TITLE).text().trim();
+}
+
+function getRecipeDetails($: CheerioSelector): RecipeDetails {
+  const matchInt = (data: string, reg: RegExp, logic?: (s: RegExpMatchArray) => number): number => {
+    let result = data.match(reg);
+    return logic ? 
+      (result ? logic(result) : 0) :
+      (result ? parseInt(result[0]) : 0)
+  };
+  
+  const parseTime = (data: string) => {
+    let parsedHoursAsMin = matchInt(data, /(\d+) hours?/, result => parseInt(result[0])*60);
+    let parsedMinutes = matchInt(data, /(\d+) minutes?/);
+    return parsedHoursAsMin + parsedMinutes;
+  };
+
+  const LAURA_MAP: {
+    [index:string] : {
+      key: string,
+      transform: (data: string) => number,
+    }
+  } = {
+    preparationTime: {
+      key: 'Preparation',
+      transform: parseTime,
+    },
+    cookingTime: {
+      key: 'Cook time',
+      transform: parseTime,
+    },
+    servings: {
+      key: 'Servings',
+      transform: (data: string) => matchInt(data, /\d+/),
+    }
+  };
+
+  let details = new RecipeDetails();
+  const detailsKey = Object.keys(details);
+  const data = $(SELECTORS.DETAILS);
+  let _rawData: any = {};
+  
+  data.each((i, el) => {
+    let key: string = $(el).find('span').text();
+    _rawData[key] = $(el).contents().last().text();
+  });
+
+  return detailsKey.reduce((recipeDetails, key) => {
+    let _key = LAURA_MAP[key];
+    return {
+      ...recipeDetails,
+      [key]: _key.transform(_rawData[_key.key])
+    }
+  }, details);
+}
+
+function getIngredients($: CheerioSelector): ComposedIngredients[] {
+  const parseIngredients = (rawIngredient: string): Ingredient => {
+    let parsed = parseIngredient(rawIngredient);
+    return {
+      name: parsed.ingredient,
+      quantity: Number(parsed.quantity) || 0,
+      unit: parsed.unit || '',
+      _original: rawIngredient,
+    };
+  };
+  const ingredientsScrape = $(SELECTORS.INGREDIENTS).children();
+  const isNewIngredientList = (tagName: string): boolean => tagName === 'span';
+  
+  let ingredients: ComposedIngredients[] = [];
+
+  return ingredientsScrape.toArray().reduce((list, rawIngredient)=>{
+    let last = list.length - 1;
+    let text = $(rawIngredient).text();
+    if (isNewIngredientList(rawIngredient.tagName)) {
+      return list.concat([{name: text, ingredients: []}]);
+    } else {
+      list[last].ingredients = list[last].ingredients.concat([parseIngredients(text)]);
+      return list;
+    }
+  }, ingredients);
+}
+
+function getInstructions($: CheerioSelector): string[] {
+  const data = $(SELECTORS.INSTRUCTIONS).text();
+  return data.split('\n').filter(s => s !== '').map(s => s.trim().replace(/^\d\)\s*/, ''));
+}
+
+const LITK_CONFIG = {
+  name: 'Laura in the Kitchen',
+  domain: 'laurainthekitchen',
+  website: LITK_AUTHOR_DATA.website,
+  scrapeRecipe: ($: CheerioSelector): Recipe => {
+    return new Recipe(
+      getRecipeName($),
+      LITK_AUTHOR_DATA,
+      getRecipeDetails($),
+      getIngredients($),
+      getInstructions($),
+    )
+  },
+}
+
+export default LITK_CONFIG;