ScavDB-Webcrawler/bot.js at master · altays/ScavDB-Webcrawler · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
const Feed = require('rss-to-json');
const MongoClient = require('mongodb');
const assert = require('assert')

if (process.env.NODE_ENV !== 'production') {
    require('dotenv').config();
}

const url = process.env.MONGO_URL;

// store all RSS feeds as objects with link to scav page

// research how to loop over objects


// run the load script for each podcast


let alabaster = {
    rssFeed: "https://pinecast.com/feed/alabasters-haberdashery",
    scavLink: "https://www.scavengersnetwork.com/alabaster"
}

let wham = {
    rssFeed: "https://feed.goshortwave.com/podcasts/wet-hot-american-moon-juice-XSIgTGPJa-XSNxHb/rss" ,
    scavLink: "https://www.scavengersnetwork.com/whamjuice"
}

let spookypod = {
    rssFeed: "https://pinecast.com/feed/spooky-spouses",
    scavLink: "https://www.scavengersnetwork.com/spookyspouses"
}

let cantWaitPod ={
    rssFeed: "https://feed.goshortwave.com/podcasts/can-t-hardly-wait-minute-rtJyNsmsjfKqeODv/rss",
    scavLink: "https://www.scavengersnetwork.com/jamesanderinminutemovies"
}

// Use connect method to connect to the Server
MongoClient.connect(url, {useNewUrlParser: true, useUnifiedTopology: true}, async function(err, client) {
    const db = client.db(process.env.MONGO_DATABASE)
    const collection = db.collection(process.env.MONGO_COLLECTION)
    try {

    assert.strictEqual(null, err);
    console.log("Connected successfully to server");

    let podArray = []

    let docs = await Feed.load(cantWait).then( rss => {
        let {items, title, description, link, category, image} = rss;

        let podcastMetaData = {
            name: rss.title,
            imageURL: rss.image,
            // description: rss.description.split("\n")[0],
            scavLink: "https://www.scavengersnetwork.com/jamesanderinminutemovies"
        }

        for (let i = 0; i < items.length; i++) {
            let entryObject = {
                ...podcastMetaData,
                episodeTitle: rss.items[i].title,
                titleShort:  rss.items[i].title.split('\n')[0].slice(0,24),
                episodeDescription:rss.items[i].description.split("\n")[0],
                episodeNumber: i+1
            }
            podArray.push(entryObject)
        }
        return podArray

    });

    // collection.find({}).toArray( async function(err, docs) {
    //     try {
    //         await console.log("Found the following records");
    //         await console.log(docs);

    //         client.close(function() {
    //             console.log("close connection")
    //         })
    //     } catch {
    //         console.log(err)
    //     }

    //   });

    collection.insertMany(docs)
        .then( async function(result,err) {
            console.log(result)
            client.close(function() {
                console.log("close connection")
            })
        }).catch(
            console.log(err)
    )

    } catch {
        console.log(err)
    }
})