Update NVD database with json data feed
"""
- import sqlite3, urllib3, shutil, gzip, re
+ import sqlite3, urllib, shutil, gzip, re
from datetime import date
BASE_URL = "https://nvd.nist.gov/feeds/json/cve/1.0/nvdcve-1.0-"
initialize_db(c)
- http = urllib3.PoolManager()
-
for year in range(YEAR_START, date.today().year + 1):
year_url = BASE_URL + str(year)
meta_url = year_url + ".meta"
json_url = year_url + ".json.gz"
# Retrieve meta last modified date
- with http.request('GET', meta_url, preload_content=False) as r:
- date_line = str(r.data.splitlines()[0])
+ with urllib.request.urlopen(meta_url) as r:
+ date_line = str(r.read().splitlines()[0])
last_modified = re.search('lastModifiedDate:(.*)', date_line).group(1)
# Compare with current db last modified date
meta = c.fetchone()
if not meta or meta[0] != last_modified:
# Update db with current year json file
- with http.request('GET', json_url, preload_content=False) as r, open(JSON_TMPFILE, 'wb') as tmpfile:
+ with urllib.request.urlopen(json_url) as r, open(JSON_TMPFILE, 'wb') as tmpfile:
shutil.copyfileobj(r, tmpfile)
with gzip.open(JSON_TMPFILE, 'rt') as jsonfile:
update_db(c, jsonfile)