From a376efc561f819cc3939a5ed84b084bee6916712 Mon Sep 17 00:00:00 2001 From: Scisco Date: Fri, 19 Feb 2016 02:51:11 -0500 Subject: [PATCH 01/10] major refactor to better handle usgs fallback --- landsat/downloader.py | 170 +++++++++++++++++++++--------------------- 1 file changed, 85 insertions(+), 85 deletions(-) diff --git a/landsat/downloader.py b/landsat/downloader.py index 441be86..f82e0ee 100644 --- a/landsat/downloader.py +++ b/landsat/downloader.py @@ -58,42 +58,49 @@ def download(self, scenes, bands=None): """ if isinstance(scenes, list): - output = {} + files = [] for scene in scenes: - # If bands are provided the image is from 2015 or later use Amazon - self.scene_interpreter(scene) - - if (bands and int(scene[12]) > 4): - if isinstance(bands, list): - # Create a folder to download the specific bands into - path = check_create_folder(join(self.download_dir, scene)) - try: - # Always grab MTL.txt if bands are specified - if 'BQA' not in bands: - bands.append('QA') - - if 'MTL' not in bands: - bands.append('MTL') - - for band in bands: - self.amazon_s3(scene, band, path) - output[scene] = 'aws' - except RemoteFileDoesntExist: - self.google_storage(scene, self.download_dir) - output[scene] = 'google' - - else: - raise Exception('Expected bands list') - else: - self.google_storage(scene, self.download_dir) - output[scene] = 'google' - - return output + + # for all scenes if bands provided, first check AWS, if the bands exist + # download them, otherwise use Google and then USGS. + try: + # if bands are not provided, directly go to Goodle and then USGS + if not isinstance(bands, list): + raise RemoteFileDoesntExist + files.append(self.amazon_s3(scene, bands)) + + except RemoteFileDoesntExist: + try: + files.append(self.google_storage(scene, self.download_dir)) + except RemoteFileDoesntExist: + files.append(self.usgs_eros(scene, self.download_dir)) + + return files else: raise Exception('Expected sceneIDs list') + def usgs_eros(self, scene, path): + """ Downloads the image from USGS """ + + # download from usgs if login information is provided + if self.usgs_user and self.usgs_pass: + try: + api_key = api.login(self.usgs_user, self.usgs_pass) + except USGSError as e: + error_tree = ElementTree.fromstring(str(e.message)) + error_text = error_tree.find("SOAP-ENV:Body/SOAP-ENV:Fault/faultstring", api.NAMESPACES).text + raise USGSInventoryAccessMissing(error_text) + + download_url = api.download('LANDSAT_8', 'EE', [scene], api_key=api_key) + if download_url: + self.output('Source: USGS EarthExplorer', normal=True, arrow=True) + return self.fetch(download_url[0], path) + + raise RemoteFileDoesntExist('%s is not available on AWS S3, Google or USGS Earth Explorer' % scene) + raise RemoteFileDoesntExist('%s is not available on AWS S3 or Google Storage' % scene) + def google_storage(self, scene, path): """ Google Storage Downloader. @@ -110,67 +117,49 @@ def google_storage(self, scene, path): :returns: Boolean """ - sat = self.scene_interpreter(scene) - filename = scene + '.tar.bz' + sat = self.scene_interpreter(scene) url = self.google_storage_url(sat) - if self.remote_file_exists(url): - return self.fetch(url, path, filename) + self.remote_file_exists(url) - else: - # download from usgs if login information is provided - if self.usgs_user and self.usgs_pass: - try: - api_key = api.login(self.usgs_user, self.usgs_pass) - except USGSError as e: - error_tree = ElementTree.fromstring(str(e.message)) - error_text = error_tree.find("SOAP-ENV:Body/SOAP-ENV:Fault/faultstring", api.NAMESPACES).text - raise USGSInventoryAccessMissing(error_text) + self.output('Source: Google Storge', normal=True, arrow=True) + return self.fetch(url, path) - download_url = api.download('LANDSAT_8', 'EE', [scene], api_key=api_key) - if download_url: - return self.fetch(download_url[0], path, filename) + def amazon_s3(self, scene, bands): + """ + Amazon S3 downloader + """ - raise RemoteFileDoesntExist('%s is not available on AWS S3, Google or USGS Earth Explorer' % filename) + sat = self.scene_interpreter(scene) - raise RemoteFileDoesntExist('%s is not available on AWS S3 or Google Storage' % filename) + # Always grab MTL.txt and QA band if bands are specified + if 'BQA' not in bands: + bands.append('QA') - def amazon_s3(self, scene, band, path): - """ - Amazon S3 downloader + if 'MTL' not in bands: + bands.append('MTL') - :param scene: - The scene ID. - :type scene: - String - :param band: - The band number. - :type band: - String, Integer - :param path: - The directory path to where the image should be stored - :type path: - String + urls = [] - :returns: - Boolean - """ - sat = self.scene_interpreter(scene) + for band in bands: + # get url for the band + url = self.amazon_s3_url(sat, band) - if band != 'MTL': - filename = '%s_B%s.TIF' % (scene, band) - else: - filename = '%s_%s.txt' % (scene, band) - url = self.amazon_s3_url(sat, filename) + # make sure it exist + self.remote_file_exists(url) + urls.append(url) - if self.remote_file_exists(url): - return self.fetch(url, path, filename) + # create folder + path = check_create_folder(join(self.download_dir, scene)) - else: - raise RemoteFileDoesntExist('%s is not available on Amazon S3' % filename) + self.output('Source: AWS S3', normal=True, arrow=True) + for url in urls: + self.fetch(url, path) + + return path - def fetch(self, url, path, filename): + def fetch(self, url, path): """ Downloads the given url. :param url: @@ -190,18 +179,26 @@ def fetch(self, url, path, filename): Boolean """ + segments = url.split('/') + filename = segments[-1] + + # remove query parameters from the filename + filename = filename.split('?')[0] + self.output('Downloading: %s' % filename, normal=True, arrow=True) + # print(join(path, filename)) + # raise Exception if exists(join(path, filename)): size = getsize(join(path, filename)) if size == self.get_remote_file_size(url): self.output('%s already exists on your system' % filename, normal=True, color='green', indent=1) - return False - fetch(url, path) + else: + fetch(url, path) self.output('stored at %s' % path, normal=True, color='green', indent=1) - return True + return join(path, filename) def google_storage_url(self, sat): """ @@ -218,7 +215,7 @@ def google_storage_url(self, sat): filename = sat['scene'] + '.tar.bz' return url_builder([self.google, sat['sat'], sat['path'], sat['row'], filename]) - def amazon_s3_url(self, sat, filename): + def amazon_s3_url(self, sat, band): """ Return an amazon s3 url the contains the scene and band provided. @@ -234,6 +231,11 @@ def amazon_s3_url(self, sat, filename): :returns: (String) The URL to a S3 file """ + if band != 'MTL': + filename = '%s_B%s.TIF' % (sat['scene'], band) + else: + filename = '%s_%s.txt' % (sat['scene'], band) + return url_builder([self.s3, sat['sat'], sat['path'], sat['row'], sat['scene'], filename]) def remote_file_exists(self, url): @@ -249,10 +251,8 @@ def remote_file_exists(self, url): """ status = requests.head(url).status_code - if status == 200: - return True - else: - return False + if status != 200: + raise RemoteFileDoesntExist def get_remote_file_size(self, url): """ Gets the filesize of a remote file. From e2d871cf831418f83011f1d7b2c33ee555bad111 Mon Sep 17 00:00:00 2001 From: Scisco Date: Fri, 19 Feb 2016 02:51:33 -0500 Subject: [PATCH 02/10] update tests to catpure downloader refactor --- tests/test_download.py | 108 ++++++++++++++++++++--------------------- 1 file changed, 53 insertions(+), 55 deletions(-) diff --git a/tests/test_download.py b/tests/test_download.py index 14ed7e7..d1ab31c 100644 --- a/tests/test_download.py +++ b/tests/test_download.py @@ -19,8 +19,8 @@ class TestDownloader(unittest.TestCase): @classmethod def setUpClass(cls): - cls.d = Downloader() cls.temp_folder = mkdtemp() + cls.d = Downloader(download_dir=cls.temp_folder) cls.scene = 'LT81360082013127LGN01' cls.scene_2 = 'LC82050312014229LGN00' cls.scene_s3 = 'LC80010092015051LGN00' @@ -47,63 +47,52 @@ def test_download(self, mock_fetch): # download one scene self.d.download([self.scene]) - self.assertEqual({self.scene: 'google'}, self.d.download([self.scene])) + paths = self.d.download([self.scene]) + self.assertTrue(isinstance(paths, list)) + self.assertEqual([self.temp_folder + '/' + self.scene + '.tar.bz'], + paths) # download multiple scenes - self.assertEqual({self.scene: 'google', self.scene_2: 'google'}, self.d.download([self.scene, self.scene_2])) + paths = self.d.download([self.scene, self.scene_2]) + test_paths = [self.temp_folder + '/' + self.scene + '.tar.bz', + self.temp_folder + '/' + self.scene_2 + '.tar.bz'] + self.assertTrue(isinstance(paths, list)) + self.assertEqual(test_paths, paths) # Test if error is raised when passing scene as string instead of list self.assertRaises(Exception, self.d.download, self.scene) # Test when passing band list along with sceneID - self.assertEqual({self.scene_s3: 'aws', self.scene_s3_2: 'aws'}, - self.d.download([self.scene_s3, self.scene_s3_2], bands=[11])) + paths = self.d.download([self.scene_s3, self.scene_s3_2], bands=[11]) + test_paths = [self.temp_folder + '/' + self.scene_s3, + self.temp_folder + '/' + self.scene_s3_2] + self.assertEqual(test_paths, paths) - # Test whether passing band as string raises an exception - self.assertRaises(Exception, self.d.download, self.scene, 4) + # When passing scene as string, google storage download should be triggered + paths = self.d.download([self.scene], bands=4) + test_paths = [self.temp_folder + '/' + self.scene + '.tar.bz'] + self.assertEqual(test_paths, paths) - @mock.patch('landsat.downloader.Downloader.amazon_s3') @mock.patch('landsat.downloader.Downloader.google_storage') - def test_download_google_amazon(self, fake_google, fake_amazon): + def test_download_google_when_amazon_is_unavailable(self, fake_google): """ Test whether google or amazon are correctly selected based on input """ - fake_amazon.return_value = True fake_google.return_value = False # Test if google is used when an image from 2014 is passed even if bands are provided self.d.download([self.scene], bands=[432]) fake_google.assert_called_with(self.scene, self.d.download_dir) - # Test if amazon is used when an image from 2015 is passed with bands - self.d.download([self.scene_s3], bands=[432]) - fake_amazon.assert_called_with(self.scene_s3, 'MTL', self.d.download_dir + '/' + self.scene_s3) - @mock.patch('landsat.downloader.fetch') - def test_google_storage(self, mock_fetch): - mock_fetch.return_value = True - - # If the file exist - self.assertTrue(self.d.google_storage(self.scene, self.temp_folder)) - - # If scene id is incorrect - self.assertRaises(IncorrectSceneId, self.d.google_storage, 'somerandomscene', self.temp_folder) - - # If scene id doesn't exist - self.assertRaises(RemoteFileDoesntExist, self.d.google_storage, 'LG21360082013227LGN01', - self.temp_folder) + def test_download_amazon_when_available(self, mock_fetch): + """ Test whether google or amazon are correctly selected based on input """ - @mock.patch('landsat.downloader.fetch') - def test_amazon_s3(self, mock_fetch): mock_fetch.return_value = True - scene = self.scene_s3 - self.assertTrue(self.d.amazon_s3(scene, 11, self.temp_folder)) - - # If scene id is incorrect - self.assertRaises(IncorrectSceneId, self.d.amazon_s3, 'somerandomscene', 11, self.temp_folder) - - # If scene id doesn't exist - self.assertRaises(RemoteFileDoesntExist, self.d.amazon_s3, 'LT81360082013127LGN01', 33, self.temp_folder) + # Test if amazon is used when an image from 2015 is passed with bands + paths = self.d.download([self.scene_s3], bands=[4, 3, 2]) + test_paths = [self.temp_folder + '/' + self.scene_s3] + self.assertEqual(test_paths, paths) @mock.patch('landsat.downloader.fetch') def test_fetch(self, mock_fetch): @@ -112,7 +101,7 @@ def test_fetch(self, mock_fetch): sat = self.d.scene_interpreter(self.scene) url = self.d.google_storage_url(sat) - self.assertTrue(self.d.fetch(url, self.temp_folder, self.scene)) + self.assertTrue(self.d.fetch(url, self.temp_folder)) def test_remote_file_size(self): @@ -130,29 +119,38 @@ def test_google_storage_url(self): def test_amazon_s3_url(self): sat = self.d.scene_interpreter(self.scene) - filename = '%s_B11.TIF' % self.scene - - string = self.d.amazon_s3_url(sat, filename) + string = self.d.amazon_s3_url(sat, 11) expect = os.path.join(S3_LANDSAT, 'L8/136/008/LT81360082013127LGN01/LT81360082013127LGN01_B11.TIF') self.assertEqual(expect, string) def test_remote_file_exist(self): - # Test a S3 url that exists - self.assertTrue(self.d.remote_file_exists(os.path.join(S3_LANDSAT, 'L8/003/017/LC80030172015001L' - 'GN00/LC80030172015001LGN00_B6.TIF'))) - - # Test a S3 url that doesn't exist - self.assertFalse(self.d.remote_file_exists(os.path.join(S3_LANDSAT, 'L8/003/017/LC80030172015001L' - 'GN00/LC80030172015001LGN00_B34.TIF'))) - - # Test a Google Storage url that doesn't exist - self.assertFalse(self.d.remote_file_exists(os.path.join(GOOGLE_STORAGE, 'L8/003/017/LC80030172015001L' - 'GN00/LC80030172015001LGN00_B6.TIF'))) - - # Test a Google Storage url that exists - self.assertTrue(self.d.remote_file_exists(os.path.join(GOOGLE_STORAGE, - 'L8/003/017/LC80030172015001LGN00.tar.bz'))) + # Exists and should return None + + self.assertIsNone(self.d.remote_file_exists(os.path.join(S3_LANDSAT, 'L8/003/017/LC80030172015001L' + 'GN00/LC80030172015001LGN00_B6.TIF'))) + + # Doesn't exist and should raise errror + with self.assertRaises(RemoteFileDoesntExist): + self.d.remote_file_exists( + os.path.join( + S3_LANDSAT, + 'L8/003/017/LC80030172015001LGN00/LC80030172015001LGN00_B34.TIF' + ) + ) + + # Doesn't exist and should raise errror + with self.assertRaises(RemoteFileDoesntExist): + self.d.remote_file_exists( + os.path.join( + GOOGLE_STORAGE, + 'L8/003/017/LC80030172015001LGN00/LC80030172015001LGN00_B6.TIF' + ) + ) + + # Exist and shouldn't raise error + self.assertIsNone(self.d.remote_file_exists(os.path.join(GOOGLE_STORAGE, + 'L8/003/017/LC80030172015001LGN00.tar.bz'))) def test_scene_interpreter(self): # Test with correct input From 723a461ffd7ac075a355026732d2a25d045b1d26 Mon Sep 17 00:00:00 2001 From: Scisco Date: Fri, 19 Feb 2016 03:06:55 -0500 Subject: [PATCH 03/10] update download logic to better handle usgs fallback --- landsat/landsat.py | 19 +++++-------------- 1 file changed, 5 insertions(+), 14 deletions(-) diff --git a/landsat/landsat.py b/landsat/landsat.py index 4ccc913..b2d3c79 100755 --- a/landsat/landsat.py +++ b/landsat/landsat.py @@ -392,23 +392,14 @@ def main(args): if not args.bands: bands = [4, 3, 2] - downloaded = d.download(args.scenes, bands) + files = d.download(args.scenes, bands) if args.process: if not args.bands: args.bands = '432' force_unzip = True if args.force_unzip else False - for scene, src in downloaded.iteritems(): - if args.dest: - path = join(args.dest, scene) - else: - path = join(settings.DOWNLOAD_DIR, scene) - - # Keep using Google if the image is before 2015 - if src == 'google': - path = path + '.tar.bz' - - stored = process_image(path, args.bands, False, args.pansharpen, args.ndvi, force_unzip, + for f in files: + stored = process_image(f, args.bands, False, args.pansharpen, args.ndvi, force_unzip, args.ndvigrey, bounds=bounds) if args.upload: @@ -467,8 +458,8 @@ def process_image(path, bands=None, verbose=False, pansharpen=False, ndvi=False, p = Simple(path, bands=bands, dst_path=settings.PROCESSED_IMAGE, verbose=verbose, force_unzip=force_unzip, bounds=bounds) - except IOError: - exit("Zip file corrupted", 1) + except IOError as e: + exit(e.message, 1) except FileDoesNotExist as e: exit(e.message, 1) From a834e22068b046a8a14a1df8b27b56fd1b56e320 Mon Sep 17 00:00:00 2001 From: Scisco Date: Fri, 19 Feb 2016 03:07:06 -0500 Subject: [PATCH 04/10] more realastic tests --- tests/test_landsat.py | 52 ++++++++++++++++++------------------------- 1 file changed, 22 insertions(+), 30 deletions(-) diff --git a/tests/test_landsat.py b/tests/test_landsat.py index 7489a82..4e046bd 100644 --- a/tests/test_landsat.py +++ b/tests/test_landsat.py @@ -132,78 +132,71 @@ def test_download_incorrect(self): ['The SceneID provided was incorrect', 1]) @mock.patch('landsat.landsat.process_image') - @mock.patch('landsat.landsat.Downloader.download') - def test_download_process_continuous(self, mock_downloader, mock_process): + @mock.patch('landsat.downloader.fetch') + def test_download_process_continuous(self, fetch, mock_process): """Test download and process commands together""" - mock_downloader.return_value = {'LC80010092015051LGN00': 'aws', - 'LC80010092014051LGN00': 'aws'} + fetch.return_value = True mock_process.return_value = 'image.TIF' - args = ['download', 'LC80010092015051LGN00', 'LC80010092014051LGN00', '-b', '432', '-d', self.mock_path, '-p'] + args = ['download', 'LC80010092015051LGN00', 'LC80470222014354LGN00', '-b', '432', '-d', self.mock_path, '-p'] output = landsat.main(self.parser.parse_args(args)) - mock_downloader.assert_called_with(['LC80010092015051LGN00', 'LC80010092014051LGN00'], [4, 3, 2]) - mock_process.assert_called_with('path/to/folder/LC80010092014051LGN00', '432', + mock_process.assert_called_with('path/to/folder/LC80470222014354LGN00', '432', False, False, False, False, False, bounds=None) self.assertEquals(output, ["The output is stored at image.TIF", 0]) # Call with force unzip flag - args = ['download', 'LC80010092015051LGN00', 'LC80010092014051LGN00', '-b', '432', '-d', + args = ['download', 'LC80010092015051LGN00', 'LC80470222014354LGN00', '-b', '432', '-d', self.mock_path, '-p', '--force-unzip'] output = landsat.main(self.parser.parse_args(args)) - mock_downloader.assert_called_with(['LC80010092015051LGN00', 'LC80010092014051LGN00'], [4, 3, 2]) - mock_process.assert_called_with('path/to/folder/LC80010092014051LGN00', '432', False, False, False, + mock_process.assert_called_with('path/to/folder/LC80470222014354LGN00', '432', False, False, False, True, False, bounds=None) self.assertEquals(output, ["The output is stored at image.TIF", 0]) # Call with pansharpen - args = ['download', 'LC80010092015051LGN00', 'LC80010092014051LGN00', '-b', '432', '-d', + args = ['download', 'LC80010092015051LGN00', 'LC80470222014354LGN00', '-b', '432', '-d', self.mock_path, '-p', '--pansharpen'] output = landsat.main(self.parser.parse_args(args)) - mock_downloader.assert_called_with(['LC80010092015051LGN00', 'LC80010092014051LGN00'], [4, 3, 2, 8]) - mock_process.assert_called_with('path/to/folder/LC80010092014051LGN00', '432', False, True, False, + mock_process.assert_called_with('path/to/folder/LC80470222014354LGN00', '432', False, True, False, False, False, bounds=None) self.assertEquals(output, ["The output is stored at image.TIF", 0]) # Call with pansharpen and clipping - args = ['download', 'LC80010092015051LGN00', 'LC80010092014051LGN00', '-b', '432', '-d', + args = ['download', 'LC80010092015051LGN00', 'LC80470222014354LGN00', '-b', '432', '-d', self.mock_path, '-p', '--pansharpen', '--clip', '"-180,-180,0,0"'] output = landsat.main(self.parser.parse_args(args)) - mock_downloader.assert_called_with(['LC80010092015051LGN00', 'LC80010092014051LGN00'], [4, 3, 2, 8]) - mock_process.assert_called_with('path/to/folder/LC80010092014051LGN00', '432', False, True, False, + mock_process.assert_called_with('path/to/folder/LC80470222014354LGN00', '432', False, True, False, False, False, bounds=[-180.0, -180.0, 0.0, 0.0]) self.assertEquals(output, ["The output is stored at image.TIF", 0]) # Call with ndvi - args = ['download', 'LC80010092015051LGN00', 'LC80010092014051LGN00', '-b', '432', '-d', + args = ['download', 'LC80010092015051LGN00', 'LC80470222014354LGN00', '-b', '432', '-d', self.mock_path, '-p', '--ndvi'] output = landsat.main(self.parser.parse_args(args)) - mock_downloader.assert_called_with(['LC80010092015051LGN00', 'LC80010092014051LGN00'], [4, 5]) - mock_process.assert_called_with('path/to/folder/LC80010092014051LGN00', '432', False, False, True, + mock_process.assert_called_with('path/to/folder/LC80470222014354LGN00', '432', False, False, True, False, False, bounds=None) self.assertEquals(output, ["The output is stored at image.TIF", 0]) # Call with ndvigrey - args = ['download', 'LC80010092015051LGN00', 'LC80010092014051LGN00', '-b', '432', '-d', + args = ['download', 'LC80010092015051LGN00', 'LC80470222014354LGN00', '-b', '432', '-d', self.mock_path, '-p', '--ndvigrey'] output = landsat.main(self.parser.parse_args(args)) - mock_downloader.assert_called_with(['LC80010092015051LGN00', 'LC80010092014051LGN00'], [4, 5]) - mock_process.assert_called_with('path/to/folder/LC80010092014051LGN00', '432', False, False, False, + mock_process.assert_called_with('path/to/folder/LC80470222014354LGN00', '432', False, False, False, False, True, bounds=None) self.assertEquals(output, ["The output is stored at image.TIF", 0]) @mock.patch('landsat.landsat.Uploader') @mock.patch('landsat.landsat.process_image') - @mock.patch('landsat.landsat.Downloader.download') - def test_download_process_continuous_with_upload(self, mock_downloader, mock_process, mock_upload): + @mock.patch('landsat.downloader.fetch') + def test_download_process_continuous_with_upload(self, fetch, mock_process, mock_upload): """Test download and process commands together""" - mock_downloader.return_value = {'LC80010092015051LGN00': 'aws'} + fetch.return_value = True mock_process.return_value = 'image.TIF' mock_upload.run.return_value = True args = ['download', 'LC80010092015051LGN00', '-b', '432', '-d', self.mock_path, '-p', '-u', '--key', 'somekey', '--secret', 'somesecret', '--bucket', 'mybucket', '--region', 'this'] output = landsat.main(self.parser.parse_args(args)) - mock_downloader.assert_called_with(['LC80010092015051LGN00'], [4, 3, 2]) + # mock_downloader.assert_called_with(['LC80010092015051LGN00'], [4, 3, 2]) mock_process.assert_called_with('path/to/folder/LC80010092015051LGN00', '432', False, False, False, False, False, bounds=None) mock_upload.assert_called_with('somekey', 'somesecret', 'this') @@ -211,16 +204,15 @@ def test_download_process_continuous_with_upload(self, mock_downloader, mock_pro self.assertEquals(output, ['The output is stored at image.TIF', 0]) @mock.patch('landsat.landsat.process_image') - @mock.patch('landsat.landsat.Downloader.download') - def test_download_process_continuous_with_wrong_args(self, mock_downloader, mock_process): + @mock.patch('landsat.downloader.fetch') + def test_download_process_continuous_with_wrong_args(self, fetch, mock_process): """Test download and process commands together""" - mock_downloader.return_value = {'LC80010092015051LGN00': 'aws'} + fetch.return_value = True mock_process.return_value = 'image.TIF' args = ['download', 'LC80010092015051LGN00', '-b', '432', '-d', self.mock_path, '-p', '-u', '--region', 'whatever'] output = landsat.main(self.parser.parse_args(args)) - mock_downloader.assert_called_with(['LC80010092015051LGN00'], [4, 3, 2]) mock_process.assert_called_with('path/to/folder/LC80010092015051LGN00', '432', False, False, False, False, False, bounds=None) self.assertEquals(output, ['Could not authenticate with AWS', 1]) From 0ade809e6cd9bc24a2eded2b1e3112447e915895 Mon Sep 17 00:00:00 2001 From: Scisco Date: Fri, 19 Feb 2016 03:07:42 -0500 Subject: [PATCH 05/10] bump up proposed version to 0.12.2 --- landsat/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/landsat/__init__.py b/landsat/__init__.py index f8d9095..92a60bd 100644 --- a/landsat/__init__.py +++ b/landsat/__init__.py @@ -1 +1 @@ -__version__ = '0.12.1' +__version__ = '0.12.2' From 49883cf85f45356f0edbc122fd996f6feec704e0 Mon Sep 17 00:00:00 2001 From: Nat Wilson Date: Sat, 5 Mar 2016 14:40:12 -0500 Subject: [PATCH 06/10] fixes issue #145 In two places, landsat-util tests for the presense of lon/lat with if lon: if lat: but this fails when lon == 0 or lat == 0 replaced with an explicit check for `None` --- landsat/landsat.py | 11 +++++++++-- landsat/search.py | 2 +- tests/test_search.py | 12 ++++++++++++ 3 files changed, 22 insertions(+), 3 deletions(-) diff --git a/landsat/landsat.py b/landsat/landsat.py index b2d3c79..0594234 100755 --- a/landsat/landsat.py +++ b/landsat/landsat.py @@ -324,8 +324,15 @@ def main(args): s = Search() try: - lat = float(args.lat) if args.lat else None - lon = float(args.lon) if args.lon else None + if args.lat is not None: + lat = float(args.lat) + else: + lat = None + + if args.lon is not None: + lon = float(args.lon) + else: + lon = None except ValueError: return ["The latitude and longitude values must be valid numbers", 1] diff --git a/landsat/search.py b/landsat/search.py index 563ca2f..6902e23 100644 --- a/landsat/search.py +++ b/landsat/search.py @@ -218,7 +218,7 @@ def query_builder(self, paths_rows=None, lat=None, lon=None, address=None, start if address: query.append(self.address_builder(address)) - elif lat and lon: + elif (lat is not None) and (lon is not None): query.append(self.lat_lon_builder(lat, lon)) if query: diff --git a/tests/test_search.py b/tests/test_search.py index 7495997..6f141b3 100644 --- a/tests/test_search.py +++ b/tests/test_search.py @@ -34,6 +34,18 @@ def test_search(self): result = self.s.search(lat=lat, lon=lon, start_date=start_date, end_date=end_date) self.assertEqual('2015-02-06', result['results'][0]['date']) + def test_search_zero_lon(self): + # Make sure that zero coordinates are handled correctly + paths_rows = '003,003' + lon = 0.0 + lat = 52.0 + start_date = '2016-01-01' + end_date = '2016-01-10' + + result = self.s.search(start_date=start_date, end_date=end_date, + lon=0.0, lat=52.0) + self.assertEqual('2016-01-06', result['results'][0]['date']) + def test_search_with_geojson(self): # TEST A REGULAR SEARCH WITH KNOWN RESULT for paths and rows From ffdc002efa75507faa708970fa738c05a556eeb3 Mon Sep 17 00:00:00 2001 From: Scisco Date: Thu, 24 Mar 2016 10:33:08 -0400 Subject: [PATCH 07/10] switch dependency to usgs closes #167 --- requirements/docker.txt | 2 +- setup.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/requirements/docker.txt b/requirements/docker.txt index 14dd3ee..6939c1e 100644 --- a/requirements/docker.txt +++ b/requirements/docker.txt @@ -8,4 +8,4 @@ boto>=2.38.0 polyline==1.1 geocoder>=1.5.1 jsonschema==2.5.1 -git+git://github.com/developmentseed/usgs@develop +usgs==0.1.9 diff --git a/setup.py b/setup.py index 3e7105d..2d28f84 100644 --- a/setup.py +++ b/setup.py @@ -35,7 +35,7 @@ def readme(): license='CCO', platforms='Posix; MacOS X; Windows', install_requires=[ - 'usgs2==0.2.0', + 'usgs==0.1.9', 'requests==2.7.0', 'python-dateutil>=2.4.2', 'numpy>=1.9.3', From bc06b19ded9fac0f29a284ed93a2046bd0c52b70 Mon Sep 17 00:00:00 2001 From: Scisco Date: Thu, 24 Mar 2016 10:50:20 -0400 Subject: [PATCH 08/10] bump up to 0.12.3 --- CHANGES.txt | 5 +++++ landsat/__init__.py | 2 +- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/CHANGES.txt b/CHANGES.txt index 9042a3a..5fd93ee 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -1,6 +1,11 @@ Changes ======= +0.12.3 (2016-02-18) +------------------ +- Fix for #167 +- Fix for #145 + 0.12.0 (2016-02-18) ------------------ - Add USGS download fallback closes #89 diff --git a/landsat/__init__.py b/landsat/__init__.py index 92a60bd..c4e914a 100644 --- a/landsat/__init__.py +++ b/landsat/__init__.py @@ -1 +1 @@ -__version__ = '0.12.2' +__version__ = '0.12.3' From 10eb6f97f50125529310ab4931279cf42d4e0fa2 Mon Sep 17 00:00:00 2001 From: Scisco Date: Thu, 24 Mar 2016 10:55:41 -0400 Subject: [PATCH 09/10] fix version to 0.12.2 --- CHANGES.txt | 2 +- landsat/__init__.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGES.txt b/CHANGES.txt index 5fd93ee..c591502 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -1,7 +1,7 @@ Changes ======= -0.12.3 (2016-02-18) +0.12.2 (2016-02-18) ------------------ - Fix for #167 - Fix for #145 diff --git a/landsat/__init__.py b/landsat/__init__.py index c4e914a..92a60bd 100644 --- a/landsat/__init__.py +++ b/landsat/__init__.py @@ -1 +1 @@ -__version__ = '0.12.3' +__version__ = '0.12.2' From 18120db50fbd5b47c4d7bf1ea6328bfb7887457f Mon Sep 17 00:00:00 2001 From: Scisco Date: Fri, 25 Mar 2016 12:47:58 -0400 Subject: [PATCH 10/10] add matplot --- setup.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 2d28f84..1688944 100644 --- a/setup.py +++ b/setup.py @@ -47,7 +47,8 @@ def readme(): 'homura>=0.1.2', 'boto>=2.38.0', 'polyline==1.1', - 'geocoder>=1.5.1' + 'geocoder>=1.5.1', + 'matplotlib==1.5.1' ], test_suite='nose.collector', tests_require=test_requirements