Refactor: Create services with basic functions and use them in the scrapes files. Use ISO8601 format for prepTime/cookTime variables.

ktchn/src/recipes/model.ts

@@ -23,17 +23,16 @@ interface Temperature {
 }
 
 export class RecipeDetails {
-  [key:string]: number|string 
-  preparationTime: number|string; // TODO: Change this type to string when all scrapes get ISO8601 duration format
-  cookingTime: number|string;
-  servings: number|string;
+  preparationTime: string;
+  cookingTime: string;
+  servings: number;
 
   constructor(
-    prepTime:number|string = 0,
-    cookingTime:number|string = 0,
-    servings:number|string = 0) {
-      this.preparationTime = Number(prepTime);
-      this.cookingTime = Number(cookingTime);
+    prepTime: string = '',
+    cookingTime: string = '',
+    servings: number|string = 0) {
+      this.preparationTime = prepTime;
+      this.cookingTime = cookingTime;
       this.servings = Number(servings);
     }
 }

ktchn/src/recipes/scrape/service.ts

@@ -0,0 +1,18 @@
+import { Recipe } from '../model';
+
+type MapFc = (i: number, element: CheerioElement) => string;
+
+export const getTextList =
+  ($: CheerioSelector) => (SELECTOR: string, mapFc?:MapFc): string[] => {
+    return $(SELECTOR).map((mapFc ? mapFc : (i, e) => $(e).text().trim())).get()
+  };
+
+export const getAttr =
+  ($: CheerioSelector) => (SELECTOR: string, ATTR: string) => $(SELECTOR).attr(ATTR);
+
+export const getText =
+  ($: CheerioSelector) => (SELECTOR: string): string => $(SELECTOR).text().trim();
+
+export const rmBreakLines = (str: string): string => str.replace(/(\n|\t|\s{2,})/g, ' ').trim();
+
+export const rmEquivalence = (str: string): string => str.replace(/\([0-9.]+ \w+\) ?/, '');

ktchn/src/recipes/scrape/sources/all-recipes.ts

@@ -1,7 +1,10 @@
 import { Recipe, Author, RecipeDetails, ComposedIngredients } from '../../model';
 import { parseIngredients } from '../index';
+import { getText, getAttr, getTextList } from './../service';
 
 const SELECTORS = {
+  TITLE: 'h1#recipe-main-content',
+  AUTHOR: 'span.submitter__name',
   PREP_TIME: 'time[itemprop="prepTime"]',
   COOK_TIME: 'time[itemprop="cookTime"]',
   SERVINGS: 'meta#metaRecipeServings',
@@ -9,23 +12,15 @@ const SELECTORS = {
   INSTRUCTIONS: 'li.step',
   SUMMARY: 'meta#metaDescription',
   TAGS: 'meta[itemprop="recipeCategory"]',
-}
-
-function getRecipeName($: CheerioSelector): string {
-  return $('h1#recipe-main-content').text();
-}
-
-function getAuthorData($: CheerioSelector): Author {
-  return {
-    name: $('span.submitter__name').text(),
-  };
+  DATETIME: 'datetime',
+  CONTENT: 'content',
 }
 
 function getRecipeDetails($: CheerioSelector): RecipeDetails {
   return {
-    preparationTime: $(SELECTORS.PREP_TIME).attr('datetime'),
-    cookingTime: $(SELECTORS.COOK_TIME).attr('datetime'),
-    servings: Number($(SELECTORS.SERVINGS).attr('content')),
+    preparationTime: getAttr($)(SELECTORS.PREP_TIME, SELECTORS.DATETIME),
+    cookingTime: getAttr($)(SELECTORS.COOK_TIME, SELECTORS.DATETIME),
+    servings: Number(getAttr($)(SELECTORS.SERVINGS, SELECTORS.CONTENT)),
   };
 }
 
@@ -53,24 +48,20 @@ function getIngredients($: CheerioSelector): ComposedIngredients[] {
   return ingredients;
 }
 
-function getInstructions($: CheerioSelector): string[] {
-  return $(SELECTORS.INSTRUCTIONS).toArray().map(element => $(element).text().trim());
-}
-
-const getTextList = (SELECTOR: string) => ($: CheerioSelector): string[] => $(SELECTOR).map((i, e) => $(e).text().trim()).get()
-
 const AR_CONFIG = {
   name: 'All Recipes',
   domain: 'allrecipes',
   website: 'https://www.allrecipes.com',
   scrapeRecipe: ($: CheerioSelector): Recipe => ({
-    name: getRecipeName($),
-    author: getAuthorData($),
+    name: getText($)(SELECTORS.TITLE),
+    author: {
+      name: getText($)(SELECTORS.AUTHOR)
+    },
     details: getRecipeDetails($),
     ingredients: getIngredients($),
-    instructions: getTextList(SELECTORS.INSTRUCTIONS)($),
+    instructions: getTextList($)(SELECTORS.INSTRUCTIONS),
     tags: $(SELECTORS.TAGS).map((i, e) => $(e).attr('content')).get(),
-    summary: $(SELECTORS.SUMMARY).attr('content'),
+    summary: getAttr($)(SELECTORS.SUMMARY, SELECTORS.CONTENT),
   }),
 }
 

ktchn/src/recipes/scrape/sources/joy-of-baking.ts

@@ -1,5 +1,6 @@
 import { Recipe, Author, RecipeDetails, ComposedIngredients } from '../../model';
 import { parseIngredients } from '../index';
+import { rmBreakLines, getText, rmEquivalence } from './../service';
 
 const JOB_AUTHOR_DATA: Author = {
   name: 'Stephanie Jaworski',
@@ -10,29 +11,24 @@ const SELECTORS = {
   TITLE: 'h1 > span.Heading',
 }
 
-function cleanText(text: string): string {
-  return text.replace(/(\n|\t|\s{2,})/g, ' ').trim();
-}
-
 function getRecipeName($: CheerioSelector): string {
-  return $(SELECTORS.TITLE).text().replace(/(.\n)?.Recipe.*/, '').trim();
+  return getText($)(SELECTORS.TITLE).replace(/(.\n)?.Recipe.*/, '').trim();
 }
 
 function getIngredients($: CheerioSelector): ComposedIngredients[] {
   const rawData = $('td [width="249"] p').toArray();
   const isIngredient = (element: CheerioElement): boolean => $(element).hasClass('ingredient');
-  const removeEquivalence = (str: string): string => str.replace(/\([0-9.]+ \w+\) ?/, '');
 
   let ingredients: ComposedIngredients[] = [];
   return rawData.reduce((list:any, element, index) => {
     let rawIngredient = $(element).text();
-    rawIngredient = cleanText(rawIngredient);
+    rawIngredient = rmBreakLines(rawIngredient);
     let last = list.length - 1;
     if(isIngredient(element)) {
       let ingredient = parseIngredients(rawIngredient);
       list[last].ingredients = list[last].ingredients.concat([{
         ...ingredient,
-        name: removeEquivalence(ingredient.name)
+        name: rmEquivalence(ingredient.name)
       }]);
       return list;
     } else {
@@ -43,7 +39,7 @@ function getIngredients($: CheerioSelector): ComposedIngredients[] {
 
 function getInstructions($: CheerioSelector): string[] {
   const rawData = $('td [style="border-top-style: none; border-top-width: medium"] p').toArray();
-  return rawData.map(element => cleanText($(element).text()));
+  return rawData.map(element => rmBreakLines($(element).text()));
 }
 
 

ktchn/src/recipes/scrape/sources/just-one-cookbook.ts

@@ -1,6 +1,7 @@
 import { Recipe, Ingredient } from "../../model";
 import { RecipeDetails, ComposedIngredients } from '../../model';
 import { parseIngredients } from '../index';
+import { getText, getTextList, rmEquivalence } from './../service';
 import R from 'ramda';
 
 const SELECTORS = {
@@ -38,8 +39,6 @@ const getRecipeDetails = ($: CheerioSelector): RecipeDetails => {
 const getIngredients = ($: CheerioSelector): ComposedIngredients[] => {
   let ingredients: ComposedIngredients[] = [{name: '', ingredients: []}];
   const rawData = $(SELECTORS.INGREDIENTS);
-  const removeEquivalence = (str: string): string => str.replace(/(\([0-9.]+ \w+\))|(\([0-9.]+ \w+\/[0-9.]+ \w+\))/, '');
-  const getString = (element: CheerioElement, selector: string): string => $(element).find(selector).text();
   const cleanName = (element: CheerioElement, not: string): string => $(element).children().not(not).toArray().map(x => $(x).text()).join(' ');
   rawData.each((i, element) => {
     const last = ingredients.length - 1;
@@ -64,22 +63,19 @@ const getIngredients = ($: CheerioSelector): ComposedIngredients[] => {
   return ingredients;
 }
 
-const getTextList = (SELECTOR: string) => ($: CheerioSelector): string[] => $(SELECTOR).map((i, e) => $(e).text()).get()
-const getText = (SELECTOR: string) => ($: CheerioSelector): string => $(SELECTOR).text().trim();
-
 const JOC_CONFIG = {
   name: 'Just One Cookbook',
   domain: 'justonecookbook',
   website: 'https://www.justonecookbook.com',
   scrapeRecipe: ($: CheerioSelector): Recipe => ({
-    name: getRecipeName($),
+    name: getText($)(SELECTORS.TITLE),
     author: { name: 'Nami' },
     details: getRecipeDetails($),
     ingredients: getIngredients($),
-    instructions: getTextList(SELECTORS.INSTRUCTIONS)($),
-    tags: (getText(SELECTORS.TAGS)($)).split(',').map(R.trim),
-    course: [getText(SELECTORS.COURSE)($)],
-    summary: getText(SELECTORS.SUMMARY)($),
+    instructions: getTextList($)(SELECTORS.INSTRUCTIONS),
+    tags: (getText($)(SELECTORS.TAGS)).split(',').map(R.trim),
+    course: [getText($)(SELECTORS.COURSE)],
+    summary: getText($)(SELECTORS.SUMMARY),
   })
 }
 

ktchn/src/recipes/scrape/sources/laura-in-the-kitchen.ts

@@ -1,5 +1,6 @@
 import { Recipe, RecipeDetails, Ingredient, ComposedIngredients, Author } from '../../model';
 import { parse as parseIngredient } from "recipe-ingredient-parser";
+import { getText, getTextList, getAttr } from "./../service";
 
 const LITK_AUTHOR_DATA: Author = {
   name: 'Laura Vitale',
@@ -8,66 +9,20 @@ const LITK_AUTHOR_DATA: Author = {
 
 const SELECTORS = {
   TITLE: '.cs-page-title > h1',
-  DETAILS: '.cs-recipe-details > div',
+  DETAILS: 'script[type="application/ld+json"]',
   INGREDIENTS: '.cs-ingredients-check-list > ul',
   INSTRUCTIONS: '#recipe-process > ul',
 }
 
-function getRecipeName($: CheerioSelector): string {
-  return $(SELECTORS.TITLE).text().trim();
-}
-
 function getRecipeDetails($: CheerioSelector): RecipeDetails {
-  const matchInt = (data: string, reg: RegExp, logic?: (s: RegExpMatchArray) => number): number => {
-    let result = data.match(reg);
-    return logic ? 
-      (result ? logic(result) : 0) :
-      (result ? parseInt(result[0]) : 0)
-  };
-  
-  const parseTime = (data: string) => {
-    let parsedHoursAsMin = matchInt(data, /(\d+) hours?/, result => parseInt(result[0])*60);
-    let parsedMinutes = matchInt(data, /(\d+) minutes?/);
-    return parsedHoursAsMin + parsedMinutes;
-  };
+  const x = $(SELECTORS.DETAILS);
+  let parsedScript = JSON.parse(($(x).html()||'').replace(/(\n|\t|\s{2,})/g, ' ').trim())
 
-  const LAURA_MAP: {
-    [index:string] : {
-      key: string,
-      transform: (data: string) => number,
-    }
-  } = {
-    preparationTime: {
-      key: 'Preparation',
-      transform: parseTime,
-    },
-    cookingTime: {
-      key: 'Cook time',
-      transform: parseTime,
-    },
-    servings: {
-      key: 'Servings',
-      transform: (data: string) => matchInt(data, /\d+/),
-    }
-  };
-
-  let details = new RecipeDetails();
-  const detailsKey = Object.keys(details);
-  const data = $(SELECTORS.DETAILS);
-  let _rawData: any = {};
-  
-  data.each((i, el) => {
-    let key: string = $(el).find('span').text();
-    _rawData[key] = $(el).contents().last().text();
-  });
-
-  return detailsKey.reduce((recipeDetails, key) => {
-    let _key = LAURA_MAP[key];
-    return {
-      ...recipeDetails,
-      [key]: _key.transform(_rawData[_key.key])
-    }
-  }, details);
+  return new RecipeDetails(
+    parsedScript.prepTime,
+    parsedScript.cookTime,
+    parsedScript.recipeYield.match(/\d/)[0],
+  );
 }
 
 function getIngredients($: CheerioSelector): ComposedIngredients[] {
@@ -98,7 +53,7 @@ function getIngredients($: CheerioSelector): ComposedIngredients[] {
 }
 
 function getInstructions($: CheerioSelector): string[] {
-  const data = $(SELECTORS.INSTRUCTIONS).text();
+  const data = getText($)(SELECTORS.INSTRUCTIONS);
   return data.split('\n').filter(s => s !== '').map(s => s.trim().replace(/^\d\)\s*/, ''));
 }
 
@@ -107,7 +62,7 @@ const LITK_CONFIG = {
   domain: 'laurainthekitchen',
   website: LITK_AUTHOR_DATA.website,
   scrapeRecipe: ($: CheerioSelector): Recipe => ({
-    name: getRecipeName($),
+    name: getText($)(SELECTORS.TITLE),
     author: LITK_AUTHOR_DATA,
     details: getRecipeDetails($),
     ingredients: getIngredients($),