MetaBase/UPDATE_PIPELINE at master · dbolser/MetaBase · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86

## This shell script is intended as a guideline for the steps needed
## to update MetaBase to include the latest version of the NAR's
## Molecular Biology Database collection [1]. It is only intended as a
## usefull overview of the steps; each step should probably be run
## manually.


YEAR=2007
YEAR=2008
YEAR=2011


## Step 1, download the webpages from NAR

mkdir Scrape-${YEAR}


## Until I can think of a better way, I blindly try to download every
## possible database accession number.

URL=http://www.oxfordjournals.org/nar/database/summary

for ACCN_NO in `seq -w 1 1500`; do
    echo wget ${URL}/${ACCN_NO} \
	--timestamping \
	--output-document Scrape-${YEAR}/db_accn_${ACCN_NO}.html \
	--append-output   Scrape-${YEAR}/wget-${YEAR}.log

done \
    > Scrape-${YEAR}/wget-${YEAR}.list

wc -l Scrape-${YEAR}/wget-${YEAR}.list

source Scrape-${YEAR}/wget-${YEAR}.list


## Drop those that dont exist

find Scrape-${YEAR}/ -size 0 -exec rm {} \;


## Step 2, compare the list of databases in MetaBase with the list we
## just downloaded.

## Step 2.1, Get (and check) the database titles we downloaded.
./get_titles_from_nar_html.plx \
  Scrape-${YEAR}/db_accn_* \
  1> Scrape-${YEAR}/titles_from_nar_html-${YEAR}.list \
  2> Scrape-${YEAR}/titles_from_nar_html-${YEAR}.warnings


## The above may throw up some 'bad' databases (that won't parse).

## 2011

# HTML::Parse: Skipping unknown tag be
Scrape-2011/db_accn_0844.html
Scrape-2011/db_accn_1151.html


## Missing address line
mv Scrape-2011/db_accn_0069.html Scrape-2011/bad_db_accn_0069.html
mv Scrape-2011/db_accn_0075.html Scrape-2011/bad_db_accn_0075.html
mv Scrape-2011/db_accn_0193.html Scrape-2011/bad_db_accn_0193.html
mv Scrape-2011/db_accn_0756.html Scrape-2011/bad_db_accn_0756.html
mv Scrape-2011/db_accn_0777.html Scrape-2011/bad_db_accn_0777.html
mv Scrape-2011/db_accn_1340.html Scrape-2011/bad_db_accn_1340.html

## Missing URL line
mv Scrape-2011/db_accn_1160.html Scrape-2011/bad_db_accn_1160.html

## URL isn't hyperlinked, unlike all the others
Scrape-2011/db_accn_1293.html # Manually fixed.


## References

# [1] Insert appropriate link to MB.