{ "name": "Advanced Web Scraping", "flow": [ { "id": 1, "module": "http:ActionSendData", "version": 3, "parameters": { "handleErrors": true, "useNewZLibDeCompress": true }, "mapper": { "ca": "", "qs": [], "url": "https://www.yelp.com/search?find_desc=Landscaping+Services&find_loc=New+York%2C+NY%2C+United+States", "gzip": true, "method": "get", "headers": [], "timeout": "", "useMtls": false, "authPass": "", "authUser": "", "bodyType": "x_www_form_urlencoded", "formFields": [], "serializeUrl": false, "shareCookies": false, "parseResponse": false, "followRedirect": true, "useQuerystring": false, "followAllRedirects": false, "rejectUnauthorized": true }, "metadata": { "designer": { "x": 0, "y": 0 }, "restore": { "expect": { "qs": { "mode": "chose" }, "method": { "mode": "chose", "label": "GET" }, "headers": { "mode": "chose" }, "bodyType": { "label": "Application/x-www-form-urlencoded" }, "formFields": { "mode": "chose" } } }, "parameters": [ { "name": "handleErrors", "type": "boolean", "label": "Evaluate all states as errors (except for 2xx and 3xx )", "required": true }, { "name": "useNewZLibDeCompress", "type": "hidden" } ], "expect": [ { "name": "url", "type": "url", "label": "URL", "required": true }, { "name": "serializeUrl", "type": "boolean", "label": "Serialize URL", "required": true }, { "name": "method", "type": "select", "label": "Method", "required": true, "validate": { "enum": [ "get", "head", "post", "put", "patch", "delete", "options" ] } }, { "name": "headers", "spec": [ { "name": "name", "type": "text", "label": "Name", "required": true }, { "name": "value", "type": "text", "label": "Value" } ], "type": "array", "label": "Headers" }, { "name": "qs", "spec": [ { "name": "name", "type": "text", "label": "Name", "required": true }, { "name": "value", "type": "text", "label": "Value" } ], "type": "array", "label": "Query String" }, { "name": "bodyType", "type": "select", "label": "Body type", "validate": { "enum": [ "raw", "x_www_form_urlencoded", "multipart_form_data" ] } }, { "name": "parseResponse", "type": "boolean", "label": "Parse response", "required": true }, { "name": "authUser", "type": "text", "label": "User name" }, { "name": "authPass", "type": "password", "label": "Password" }, { "name": "timeout", "type": "uinteger", "label": "Timeout", "validate": { "max": 300, "min": 1 } }, { "name": "shareCookies", "type": "boolean", "label": "Share cookies with other HTTP modules", "required": true }, { "name": "ca", "type": "cert", "label": "Self-signed certificate" }, { "name": "rejectUnauthorized", "type": "boolean", "label": "Reject connections that are using unverified (self-signed) certificates", "required": true }, { "name": "followRedirect", "type": "boolean", "label": "Follow redirect", "required": true }, { "name": "useQuerystring", "type": "boolean", "label": "Disable serialization of multiple same query string keys as arrays", "required": true }, { "name": "gzip", "type": "boolean", "label": "Request compressed content", "required": true }, { "name": "useMtls", "type": "boolean", "label": "Use Mutual TLS", "required": true }, { "name": "formFields", "spec": [ { "name": "key", "type": "text", "label": "Key", "required": true }, { "name": "value", "type": "text", "label": "Value" } ], "type": "array", "label": "Fields" }, { "name": "followAllRedirects", "type": "boolean", "label": "Follow all redirect", "required": true } ] } }, { "id": 2, "module": "regexp:HTMLToText", "version": 1, "parameters": {}, "mapper": { "html": "{{1.data}}", "newline": "lf", "uppercaseHeadings": true }, "metadata": { "designer": { "x": 300, "y": 0 }, "restore": { "expect": { "newline": { "label": "LF (\\n) - Unix/Mac OS X" } } }, "expect": [ { "name": "html", "type": "text", "label": "HTML" }, { "name": "newline", "type": "select", "label": "Line break", "required": true, "validate": { "enum": [ "lf", "crlf", "cr" ] } }, { "name": "uppercaseHeadings", "type": "boolean", "label": "Uppercase headings", "required": true } ] } }, { "id": 3, "module": "regexp:HTMLParser", "version": 1, "parameters": { "continueWhenNoRes": true }, "mapper": { "html": "{{1.data}}", "elementType": "a" }, "metadata": { "designer": { "x": 600, "y": 0 }, "restore": { "expect": { "elementType": { "label": "Link (a)" } } }, "parameters": [ { "name": "continueWhenNoRes", "type": "boolean", "label": "Continue the execution of the route even if the module finds no matches", "required": true } ], "expect": [ { "name": "elementType", "type": "select", "label": "Element type", "required": true, "validate": { "enum": [ "img", "a", "iframe" ] } }, { "name": "html", "type": "text", "label": "HTML" } ], "interface": [ { "name": "element", "type": "text", "label": "Element" }, { "name": "innerContent", "type": "text", "label": "Inner content" }, { "name": "attrs", "spec": [ { "name": "href", "type": "text", "label": "href" }, { "name": "rel", "type": "text", "label": "rel" }, { "name": "target", "type": "text", "label": "target" } ], "type": "collection", "label": "Attributes" } ] } }, { "id": 4, "module": "builtin:BasicFeeder", "version": 1, "parameters": {}, "filter": { "name": "Filter", "conditions": [ [ { "a": "{{3.attrs.href}}", "b": "/biz/", "o": "text:contain" }, { "a": "{{3.attrs.href}}", "b": "?hrid=", "o": "text:contain" } ] ] }, "mapper": { "array": "{{3.attrs.href}}" }, "metadata": { "designer": { "x": 900, "y": 0 }, "restore": { "expect": { "array": { "mode": "edit" } } }, "expect": [ { "mode": "edit", "name": "array", "spec": [], "type": "array", "label": "Array" } ] } }, { "id": 6, "module": "util:FunctionSleep", "version": 1, "parameters": {}, "filter": { "name": "Filter", "conditions": [] }, "mapper": { "duration": "1" }, "metadata": { "designer": { "x": 1200, "y": 0 }, "restore": {}, "expect": [ { "name": "duration", "type": "uinteger", "label": "Delay", "required": true, "validate": { "max": 300, "min": 1 } } ] } }, { "id": 5, "module": "http:ActionSendData", "version": 3, "parameters": { "handleErrors": true, "useNewZLibDeCompress": true }, "mapper": { "ca": "", "qs": [], "url": "https://www.yelp.com{{4.value}}", "gzip": true, "method": "get", "headers": [], "timeout": "", "useMtls": false, "authPass": "", "authUser": "", "bodyType": "x_www_form_urlencoded", "formFields": [], "serializeUrl": false, "shareCookies": false, "parseResponse": false, "followRedirect": true, "useQuerystring": false, "followAllRedirects": false, "rejectUnauthorized": true }, "metadata": { "designer": { "x": 1500, "y": 0 }, "restore": { "expect": { "qs": { "mode": "chose" }, "method": { "mode": "chose", "label": "GET" }, "headers": { "mode": "chose" }, "bodyType": { "label": "Application/x-www-form-urlencoded" }, "formFields": { "mode": "chose" } } }, "parameters": [ { "name": "handleErrors", "type": "boolean", "label": "Evaluate all states as errors (except for 2xx and 3xx )", "required": true }, { "name": "useNewZLibDeCompress", "type": "hidden" } ], "expect": [ { "name": "url", "type": "url", "label": "URL", "required": true }, { "name": "serializeUrl", "type": "boolean", "label": "Serialize URL", "required": true }, { "name": "method", "type": "select", "label": "Method", "required": true, "validate": { "enum": [ "get", "head", "post", "put", "patch", "delete", "options" ] } }, { "name": "headers", "spec": [ { "name": "name", "type": "text", "label": "Name", "required": true }, { "name": "value", "type": "text", "label": "Value" } ], "type": "array", "label": "Headers" }, { "name": "qs", "spec": [ { "name": "name", "type": "text", "label": "Name", "required": true }, { "name": "value", "type": "text", "label": "Value" } ], "type": "array", "label": "Query String" }, { "name": "bodyType", "type": "select", "label": "Body type", "validate": { "enum": [ "raw", "x_www_form_urlencoded", "multipart_form_data" ] } }, { "name": "parseResponse", "type": "boolean", "label": "Parse response", "required": true }, { "name": "authUser", "type": "text", "label": "User name" }, { "name": "authPass", "type": "password", "label": "Password" }, { "name": "timeout", "type": "uinteger", "label": "Timeout", "validate": { "max": 300, "min": 1 } }, { "name": "shareCookies", "type": "boolean", "label": "Share cookies with other HTTP modules", "required": true }, { "name": "ca", "type": "cert", "label": "Self-signed certificate" }, { "name": "rejectUnauthorized", "type": "boolean", "label": "Reject connections that are using unverified (self-signed) certificates", "required": true }, { "name": "followRedirect", "type": "boolean", "label": "Follow redirect", "required": true }, { "name": "useQuerystring", "type": "boolean", "label": "Disable serialization of multiple same query string keys as arrays", "required": true }, { "name": "gzip", "type": "boolean", "label": "Request compressed content", "required": true }, { "name": "useMtls", "type": "boolean", "label": "Use Mutual TLS", "required": true }, { "name": "formFields", "spec": [ { "name": "key", "type": "text", "label": "Key", "required": true }, { "name": "value", "type": "text", "label": "Value" } ], "type": "array", "label": "Fields" }, { "name": "followAllRedirects", "type": "boolean", "label": "Follow all redirect", "required": true } ] } }, { "id": 7, "module": "regexp:HTMLToText", "version": 1, "parameters": {}, "mapper": { "html": "{{5.data}}", "newline": "lf", "uppercaseHeadings": true }, "metadata": { "designer": { "x": 1800, "y": 0 }, "restore": { "expect": { "newline": { "label": "LF (\\n) - Unix/Mac OS X" } } }, "expect": [ { "name": "html", "type": "text", "label": "HTML" }, { "name": "newline", "type": "select", "label": "Line break", "required": true, "validate": { "enum": [ "lf", "crlf", "cr" ] } }, { "name": "uppercaseHeadings", "type": "boolean", "label": "Uppercase headings", "required": true } ] } }, { "id": 8, "module": "openai-gpt-3:CreateCompletion", "version": 1, "parameters": { "__IMTCONN__": 1504129 }, "mapper": { "model": "gpt-4o", "top_p": "1", "select": "chat", "messages": [ { "role": "system", "content": "Please find all the services the yelp listing provides. Provide me the information in a list format, with no additional information. Keep it brief please. Here is the website:\n{{7.text}}" } ], "max_tokens": "200", "temperature": "1", "n_completions": "1", "response_format": "text" }, "metadata": { "designer": { "x": 2100, "y": 0 }, "restore": { "expect": { "stop": { "mode": "chose" }, "model": { "mode": "chose", "label": "gpt-4o (system)" }, "select": { "label": "Create a Chat Completion (GPT Models)" }, "messages": { "mode": "chose", "items": [ { "role": { "mode": "chose", "label": "System" } } ] }, "logit_bias": { "mode": "chose" }, "response_format": { "mode": "chose", "label": "Text" }, "additionalParameters": { "mode": "chose" } }, "parameters": { "__IMTCONN__": { "data": { "scoped": "true", "connection": "openai-gpt-3" }, "label": "My OpenAI connection" } } }, "parameters": [ { "name": "__IMTCONN__", "type": "account:openai-gpt-3", "label": "Connection", "required": true } ], "expect": [ { "name": "select", "type": "select", "label": "Select Method", "required": true, "validate": { "enum": [ "chat", "prompt" ] } }, { "name": "max_tokens", "type": "number", "label": "Max Tokens", "required": true }, { "name": "temperature", "type": "number", "label": "Temperature", "validate": { "max": 2, "min": 0 } }, { "name": "top_p", "type": "number", "label": "Top P", "validate": { "max": 1, "min": 0 } }, { "name": "n_completions", "type": "number", "label": "Number" }, { "name": "frequency_penalty", "type": "number", "label": "Frequency Penalty", "validate": { "max": 2, "min": -2 } }, { "name": "presence_penalty", "type": "number", "label": "Presence Penalty", "validate": { "max": 2, "min": -2 } }, { "name": "logit_bias", "spec": { "name": "value", "spec": [ { "name": "token", "type": "text", "label": "Token ID", "required": true }, { "name": "probability", "type": "number", "label": "Probability", "required": true, "validate": { "max": 100, "min": -100 } } ], "type": "collection", "label": "Token Probability" }, "type": "array", "label": "Token Probability" }, { "name": "response_format", "type": "select", "label": "Response Format", "validate": { "enum": [ "text", "json_object" ] } }, { "name": "seed", "type": "integer", "label": "Seed" }, { "name": "stop", "spec": { "name": "value", "type": "text", "label": "Stop Sequence" }, "type": "array", "label": "Stop Sequences", "validate": { "maxItems": 4 } }, { "name": "additionalParameters", "spec": { "name": "value", "spec": [ { "name": "key", "type": "text", "label": "Parameter Name", "required": true }, { "name": "type", "type": "select", "label": "Input Type", "options": [ { "label": "Text", "value": "text", "nested": [ { "name": "value", "type": "text", "label": "Parameter Value" } ], "default": true }, { "label": "Number", "value": "number", "nested": [ { "name": "value", "type": "number", "label": "Parameter Value" } ] }, { "label": "Boolean", "value": "boolean", "nested": [ { "name": "value", "type": "boolean", "label": "Parameter Value" } ] }, { "label": "Date", "value": "date", "nested": [ { "name": "value", "type": "date", "label": "Parameter Value" } ] }, { "label": "Any", "value": "any", "nested": [ { "name": "value", "type": "any", "label": "Parameter Value" } ] } ] } ], "type": "collection", "label": "Input Parameter" }, "type": "array", "label": "Other Input Parameters" }, { "name": "model", "type": "select", "label": "Model", "required": true }, { "name": "messages", "spec": { "name": "value", "spec": [ { "name": "role", "type": "select", "label": "Role", "options": [ { "label": "System", "value": "system" }, { "label": "User", "value": "user" }, { "label": "Assistant", "value": "assistant" } ], "required": true }, { "name": "content", "type": "text", "label": "Message Content" } ], "type": "collection", "label": "Message" }, "type": "array", "label": "Messages", "required": true } ] } }, { "id": 9, "module": "regexp:AdvancedParser", "version": 1, "parameters": { "global": false, "multiline": false, "sensitive": true, "singleline": false, "advancedParser": true, "continueWhenNoRes": true }, "mapper": { "text": "{{7.text}}", "pattern": "Loading interface\\.\\.\\. \\[(.*?)\\]" }, "metadata": { "designer": { "x": 2400, "y": 0 }, "restore": {}, "parameters": [ { "name": "global", "type": "boolean", "label": "Global match", "required": true }, { "name": "sensitive", "type": "boolean", "label": "Case sensitive", "required": true }, { "name": "multiline", "type": "boolean", "label": "Multiline", "required": true }, { "name": "singleline", "type": "boolean", "label": "Singleline", "required": true }, { "name": "continueWhenNoRes", "type": "boolean", "label": "Continue the execution of the route even if the module finds no matches", "required": true }, { "name": "advancedParser", "type": "hidden" } ], "expect": [ { "name": "pattern", "type": "text", "label": "Pattern", "required": true }, { "name": "text", "type": "text", "label": "Text", "required": true } ] } }, { "id": 10, "module": "regexp:AdvancedParser", "version": 1, "parameters": { "global": false, "multiline": false, "sensitive": true, "singleline": false, "advancedParser": true, "continueWhenNoRes": true }, "mapper": { "text": "{{7.text}}", "pattern": "\\(\\d{3}\\) \\d{3}-\\d{4}" }, "metadata": { "designer": { "x": 2694, "y": 4 }, "restore": {}, "parameters": [ { "name": "global", "type": "boolean", "label": "Global match", "required": true }, { "name": "sensitive", "type": "boolean", "label": "Case sensitive", "required": true }, { "name": "multiline", "type": "boolean", "label": "Multiline", "required": true }, { "name": "singleline", "type": "boolean", "label": "Singleline", "required": true }, { "name": "continueWhenNoRes", "type": "boolean", "label": "Continue the execution of the route even if the module finds no matches", "required": true }, { "name": "advancedParser", "type": "hidden" } ], "expect": [ { "name": "pattern", "type": "text", "label": "Pattern", "required": true }, { "name": "text", "type": "text", "label": "Text", "required": true } ] } }, { "id": 11, "module": "regexp:AdvancedParser", "version": 1, "parameters": { "global": false, "multiline": false, "sensitive": true, "singleline": false, "advancedParser": true, "continueWhenNoRes": true }, "mapper": { "text": "{{7.text}}", "pattern": "PEOPLE FOUND (.*?) BY SEARCHING FOR" }, "metadata": { "designer": { "x": 2993, "y": 3 }, "restore": {}, "parameters": [ { "name": "global", "type": "boolean", "label": "Global match", "required": true }, { "name": "sensitive", "type": "boolean", "label": "Case sensitive", "required": true }, { "name": "multiline", "type": "boolean", "label": "Multiline", "required": true }, { "name": "singleline", "type": "boolean", "label": "Singleline", "required": true }, { "name": "continueWhenNoRes", "type": "boolean", "label": "Continue the execution of the route even if the module finds no matches", "required": true }, { "name": "advancedParser", "type": "hidden" } ], "expect": [ { "name": "pattern", "type": "text", "label": "Pattern", "required": true }, { "name": "text", "type": "text", "label": "Text", "required": true } ] } }, { "id": 12, "module": "google-sheets:addRow", "version": 2, "parameters": { "__IMTCONN__": 3193914 }, "mapper": { "mode": "fromAll", "values": { "0": "{{11.value}}", "2": "{{10.`__IMTMATCH__`}}", "3": "{{8.choices[].message.content}}", "4": "https://yelp.com{{9.value}}", "6": "{{formatDate(now; \"DD MMM YYYY\")}}" }, "sheetId": "Sheet1", "spreadsheetId": "1MT1daiOIr0FPn8XNXZxOXXq-TaAtSFkHmvbsc6KggSM", "includesHeaders": true, "insertDataOption": "INSERT_ROWS", "valueInputOption": "USER_ENTERED", "insertUnformatted": false }, "metadata": { "designer": { "x": 3293, "y": 3 }, "restore": { "expect": { "mode": { "label": "Select from all" }, "sheetId": { "mode": "chose", "label": "Sheet1" }, "includesHeaders": { "label": "Yes", "nested": [ { "name": "values", "spec": [ { "name": "0", "type": "text", "label": "Company (A)" }, { "name": "1", "type": "text", "label": "Email (B)" }, { "name": "2", "type": "text", "label": "Phone (C)" }, { "name": "3", "type": "text", "label": "Services (D)" }, { "name": "4", "type": "text", "label": "Website (E)" }, { "name": "5", "type": "text", "label": "Context (F)" }, { "name": "6", "type": "text", "label": "Date (G)" }, { "name": "7", "type": "text", "label": "(H)" }, { "name": "8", "type": "text", "label": "(I)" }, { "name": "9", "type": "text", "label": "(J)" }, { "name": "10", "type": "text", "label": "(K)" }, { "name": "11", "type": "text", "label": "(L)" }, { "name": "12", "type": "text", "label": "(M)" }, { "name": "13", "type": "text", "label": "(N)" }, { "name": "14", "type": "text", "label": "(O)" }, { "name": "15", "type": "text", "label": "(P)" }, { "name": "16", "type": "text", "label": "(Q)" }, { "name": "17", "type": "text", "label": "(R)" }, { "name": "18", "type": "text", "label": "(S)" }, { "name": "19", "type": "text", "label": "(T)" }, { "name": "20", "type": "text", "label": "(U)" }, { "name": "21", "type": "text", "label": "(V)" }, { "name": "22", "type": "text", "label": "(W)" }, { "name": "23", "type": "text", "label": "(X)" }, { "name": "24", "type": "text", "label": "(Y)" }, { "name": "25", "type": "text", "label": "(Z)" } ], "type": "collection", "label": "Values" } ] }, "insertDataOption": { "mode": "chose", "label": "Insert rows" }, "valueInputOption": { "mode": "chose", "label": "User entered" }, "insertUnformatted": { "mode": "chose" } }, "parameters": { "__IMTCONN__": { "data": { "scoped": "true", "connection": "google" }, "label": "My Google connection (jccatliff@gmail.com)" } } }, "parameters": [ { "name": "__IMTCONN__", "type": "account:google", "label": "Connection", "required": true } ], "expect": [ { "name": "mode", "type": "select", "label": "Choose a Method", "required": true, "validate": { "enum": [ "select", "fromAll", "map" ] } }, { "name": "insertUnformatted", "type": "boolean", "label": "Unformatted", "required": true }, { "name": "valueInputOption", "type": "select", "label": "Value input option", "validate": { "enum": [ "USER_ENTERED", "RAW" ] } }, { "name": "insertDataOption", "type": "select", "label": "Insert data option", "validate": { "enum": [ "INSERT_ROWS", "OVERWRITE" ] } }, { "name": "spreadsheetId", "type": "text", "label": "Spreadsheet ID", "required": true }, { "name": "mode", "type": "select", "label": "Choose a Method", "required": true, "validate": { "enum": [ "select", "fromAll", "map" ] } }, { "name": "insertUnformatted", "type": "boolean", "label": "Unformatted", "required": true }, { "name": "valueInputOption", "type": "select", "label": "Value input option", "validate": { "enum": [ "USER_ENTERED", "RAW" ] } }, { "name": "insertDataOption", "type": "select", "label": "Insert data option", "validate": { "enum": [ "INSERT_ROWS", "OVERWRITE" ] } }, { "name": "spreadsheetId", "type": "text", "label": "Spreadsheet ID", "required": true }, { "name": "sheetId", "type": "select", "label": "Sheet Name", "required": true }, { "name": "includesHeaders", "type": "select", "label": "Table contains headers", "required": true, "validate": { "enum": [ true, false ] } }, { "name": "values", "spec": [ { "name": "0", "type": "text", "label": "Company (A)" }, { "name": "1", "type": "text", "label": "Email (B)" }, { "name": "2", "type": "text", "label": "Phone (C)" }, { "name": "3", "type": "text", "label": "Services (D)" }, { "name": "4", "type": "text", "label": "Website (E)" }, { "name": "5", "type": "text", "label": "Context (F)" }, { "name": "6", "type": "text", "label": "Date (G)" }, { "name": "7", "type": "text", "label": "(H)" }, { "name": "8", "type": "text", "label": "(I)" }, { "name": "9", "type": "text", "label": "(J)" }, { "name": "10", "type": "text", "label": "(K)" }, { "name": "11", "type": "text", "label": "(L)" }, { "name": "12", "type": "text", "label": "(M)" }, { "name": "13", "type": "text", "label": "(N)" }, { "name": "14", "type": "text", "label": "(O)" }, { "name": "15", "type": "text", "label": "(P)" }, { "name": "16", "type": "text", "label": "(Q)" }, { "name": "17", "type": "text", "label": "(R)" }, { "name": "18", "type": "text", "label": "(S)" }, { "name": "19", "type": "text", "label": "(T)" }, { "name": "20", "type": "text", "label": "(U)" }, { "name": "21", "type": "text", "label": "(V)" }, { "name": "22", "type": "text", "label": "(W)" }, { "name": "23", "type": "text", "label": "(X)" }, { "name": "24", "type": "text", "label": "(Y)" }, { "name": "25", "type": "text", "label": "(Z)" } ], "type": "collection", "label": "Values" } ] } } ], "metadata": { "instant": false, "version": 1, "scenario": { "roundtrips": 1, "maxErrors": 3, "autoCommit": true, "autoCommitTriggerLast": true, "sequential": false, "slots": null, "confidential": false, "dataloss": false, "dlq": false, "freshVariables": false }, "designer": { "orphans": [] }, "zone": "us1.make.com" } }