Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

megares_fix #336

Open
wants to merge 1 commit into
base: master
Choose a base branch
from
Open

megares_fix #336

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 8 additions & 4 deletions ariba/megares_data_finder.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,8 +8,9 @@ class Error (Exception): pass

class MegaresDataFinder:
def __init__(self, version=None):
self.url_root = 'https://megares.meglab.org/download/'
self.index_url = self.url_root + 'index.php'
self.url_root = 'https://www.meglab.org/megares/download'
self.url_root_downloads='https://www.meglab.org'
self.index_url = self.url_root
self.version = version


Expand All @@ -30,7 +31,7 @@ def _zips_from_index_page_string(cls, html_text):
except:
raise Error('Error parsing contents of megares download page. Cannot continue')

prefix = 'megares_v'
prefix = '/downloads/megares_v'
suffix = '.zip'
zips = {}

Expand All @@ -39,7 +40,9 @@ def _zips_from_index_page_string(cls, html_text):
if href.startswith(prefix) and href.endswith(suffix):
version = href[len(prefix):-len(suffix)]
zips[version] = href


#print(zips)
return zips


Expand All @@ -48,6 +51,7 @@ def _get_url_for_version(cls, zips, version=None):
if version is None:
versions = list(zips.keys())
versions.sort(key=LooseVersion)
#print('This is v',zips[versions[-1]])
return zips[versions[-1]]
else:
try:
Expand All @@ -63,6 +67,6 @@ def run(self):
zips = MegaresDataFinder._zips_from_index_page_string(html_text)
print('Found versions: ', ', '.join(list(zips.keys())))
url = MegaresDataFinder._get_url_for_version(zips, version=self.version)
return self.url_root + url
return self.url_root_downloads + url


3 changes: 2 additions & 1 deletion ariba/megares_zip_parser.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -88,13 +88,14 @@ def _load_header_mappings_file(cls, infile):
return MegaresZipParser._csv_to_dict(infile, '\t', {'Source_Database', 'MEGARes_Header', 'Source_Headers(space_separated)'}, 'MEGARes_Header')
else:
assert infile.endswith(".csv")
data = MegaresZipParser._csv_to_dict(infile, ',', {'Database', 'MEGARes_v2_header', 'Source_header'}, 'MEGARes_v2_header')
data = MegaresZipParser._csv_to_dict(infile, ',', {'Database','Source_header','MEGARes_header','UpdatedHeader'}, 'MEGARes_header')
fixed_data = {}
for key, d in data.items():
fixed_data[key] = {
"Source_Database": d["Database"],
"Source_Headers(space_separated)": d["Source_header"]
}
#print(fixed_data)
return fixed_data


Expand Down
16 changes: 9 additions & 7 deletions ariba/tests/megares_data_finder_test.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,21 +16,23 @@ def test_zips_from_index_page_string(self):
</head>

<ul>
<li><a href="megares_v1.01.zip">All Files</a></li>
<li><a href="foo.zip">All Files</a></li>
<li><a href="megares_v1.00.zip">All Files</a></li>
<li><a href="/downloads/megares_v3.00.zip">All Files</a> (.zip)</li>
<li><a href="/downloads/megares_v2.00.zip">All Files</a> (.zip)</li>
<li><a href="/downloads/megares_v1.01.zip">All Files</a> (.zip)</li>
<li><a href="/downloads/megares_v1.00.zip">All Files</a> (.zip)</li>

</html>'''

expected = {'1.00': 'megares_v1.00.zip', '1.01': 'megares_v1.01.zip'}
expected = {'3.00': '/downloads/megares_v3.00.zip', '2.00': '/downloads/megares_v2.00.zip', '1.01': '/downloads/megares_v1.01.zip', '1.00': '/downloads/megares_v1.00.zip'}
got = megares_data_finder.MegaresDataFinder._zips_from_index_page_string(html_string)
self.assertEqual(expected, got)


def test_get_url_for_version(self):
'''test _get_url_for_version'''
zips = {'1.00': 'megares_v1.00.zip', '1.01': 'megares_v1.01.zip'}
self.assertEqual('megares_v1.01.zip', megares_data_finder.MegaresDataFinder._get_url_for_version(zips))
self.assertEqual('megares_v1.00.zip', megares_data_finder.MegaresDataFinder._get_url_for_version(zips, version='1.00'))
zips = {'3.00': '/downloads/megares_v3.00.zip', '2.00': '/downloads/megares_v2.00.zip', '1.01': '/downloads/megares_v1.01.zip', '1.00': '/downloads/megares_v1.00.zip'}
self.assertEqual('/downloads/megares_v3.00.zip', megares_data_finder.MegaresDataFinder._get_url_for_version(zips))
self.assertEqual('/downloads/megares_v2.00.zip', megares_data_finder.MegaresDataFinder._get_url_for_version(zips, version='2.00'))
with self.assertRaises(megares_data_finder.Error):
self.assertEqual('megares_v1.00.zip', megares_data_finder.MegaresDataFinder._get_url_for_version(zips, version='0.42'))

14 changes: 7 additions & 7 deletions ariba/tests/megares_zip_parser_test.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,11 +14,11 @@ def test_extract_files_ok(self):
zip_file = os.path.join(data_dir, 'megares_zip_parse_extract_files_ok.zip')
tmp_dir = 'tmp.test_megares_extract_files_ok'
got = megares_zip_parser.MegaresZipParser._extract_files(zip_file, tmp_dir)
common_dir = os.path.join('megares_zip_parse_extract_files_ok', 'megares_v1.01')
common_dir = os.path.join('megares_zip_parse_extract_files_ok', 'megares_v3.00')
expected = {
'annotations': os.path.join(common_dir, 'megares_annotations_v1.01.csv'),
'fasta': os.path.join(common_dir, 'megares_database_v1.01.fasta'),
'header_mappings': os.path.join(common_dir, 'megares_to_external_header_mappings_v1.01.tsv')
'annotations': os.path.join(common_dir, 'megares_annotations_v3.00.csv'),
'fasta': os.path.join(common_dir, 'megares_database_v3.00.fasta'),
'header_mappings': os.path.join(common_dir, 'megares_to_external_header_mappings_v3.00.csv')
}

self.assertEqual(expected, got)
Expand Down Expand Up @@ -63,9 +63,9 @@ def test_load_header_mappings_file(self):

def test_write_files(self):
'''test _write_files'''
fasta_file = os.path.join(data_dir, 'megares_zip_parser_write_files', 'megares_database_v1.01.fasta')
annotations_file = os.path.join(data_dir, 'megares_zip_parser_write_files', 'megares_annotations_v1.01.csv')
mappings_file = os.path.join(data_dir, 'megares_zip_parser_write_files', 'megares_to_external_header_mappings_v1.01.tsv')
fasta_file = os.path.join(data_dir, 'megares_zip_parser_write_files', 'megares_database_v3.00.fasta')
annotations_file = os.path.join(data_dir, 'megares_zip_parser_write_files', 'megares_annotations_v3.00.csv')
mappings_file = os.path.join(data_dir, 'megares_zip_parser_write_files', 'megares_to_external_header_mappings_v3.00.csv')
sequences = {}
pyfastaq.tasks.file_to_dict(fasta_file, sequences)
annotation_data = megares_zip_parser.MegaresZipParser._load_annotations_file(annotations_file)
Expand Down