Change from single input to list

#1

Hello,

Any ideas on how I can take the below (modified from DNS example) and change it from a single URL input to a list?

const Apify = require('apify');
const { URL } = require('url');
const { promisify } = require('util');
const { resolveMx } = require('dns');

const { log, dir } = console;
const [dnsResolveMx] = [
  resolveMx,
].map(promisify);

Apify.main(async () => {
  const input = await Apify.getValue('INPUT');
  log(input);

  const { url } = input;
  if (!url) throw Error('No input URL');
  log('Input URL:', url);

  const protocol = 'http://';
  const parsedUrl = `${/http/.test(url) ? '' : protocol}${url}`;

  const { host, origin } = new URL(parsedUrl);
  log('URL host:', host);
  
  const output = { host };

  try {
    Object.assign(output, {
      hostResolve: await dnsResolveMx(host),
    });
  } catch (error) {
    log(error.message);
    Object.assign(output, { error: error.message });
}

  log('Output result:');
  dir(output);
  await Apify.setValue('OUTPUT', output);
});
#2

Hi there!

try:

const Apify = require('apify');
const { URL } = require('url');
const { promisify } = require('util');
const { resolveMx } = require('dns');

const { log, dir } = console;
const [dnsResolveMx] = [
  resolveMx,
].map(promisify);

Apify.main(async () => {
    const input = await Apify.getValue('INPUT');
    log('Input:', input);
    const { urls } = input;
    if (!urls || !urls.length) throw Error('No input URLs');
    log('Input URLS:', urls);
    const output = {};

    for (let index in urls) {
        const url = urls[index];
        const protocol = 'http://';
        const parsedUrl = `${/http/.test(url) ? '' : protocol}${url}`;

        const { host, origin } = new URL(parsedUrl);
        log('URL host:', host);

        output[host] = {};

        try {
            Object.assign(output[host], {
                hostResolve: await dnsResolveMx(host),
            });
        } catch (error) {
            log(error.message);
            Object.assign(output[host], { error: error.message });
        }
    }

    log('Output result:');
    dir(output);
    await Apify.setValue('OUTPUT', output);
});

This accepts following input:

{ "urls": ["http://example.com", "https://eatrocks.cz", "https://wired.com"] }

Best,
Marek

#3

Really appreciate you taking a look at this, I ran it today and the input side seems to work fine however the output is not as expected.

2018-02-01T22:54:29.632Z Starting Docker container
2018-02-01T22:54:30.806Z Input: { urls: [ 'http://hp.com', 'https://eatrocks.cz', 'https://wired.com' ] }
2018-02-01T22:54:30.808Z Input URLS: [ 'http://hp.com', 'https://eatrocks.cz', 'https://wired.com' ]
2018-02-01T22:54:30.810Z URL host: hp.com
2018-02-01T22:54:30.993Z URL host: eatrocks.cz
2018-02-01T22:54:31.024Z URL host: wired.com
2018-02-01T22:54:31.039Z Output result:
2018-02-01T22:54:31.044Z { 'hp.com': { hostResolve: [ [Object] ] },
2018-02-01T22:54:31.046Z   'eatrocks.cz': 
2018-02-01T22:54:31.048Z    { hostResolve: [ [Object], [Object], [Object], [Object], [Object] ] },
2018-02-01T22:54:31.049Z   'wired.com': { hostResolve: [ [Object], [Object] ] } }
#4

The output in log is displayed using console.dir() and it doesn’t show the full object (nested objects are replaced by [Object]. Try to look into “key-value store” tab in act run console and open OUTPUT key.

Or replace dir(output); with log(JSON.stringify(output, null, 2) and it will show the full output in console.

#5

Awesome, thanks for that - works a treat!