From 7ca209111a8efbcd7ff474359f0571f57f691384 Mon Sep 17 00:00:00 2001 From: shvngisingh Date: Mon, 6 Jul 2026 13:13:20 +0000 Subject: [PATCH 1/2] updated process.py script for getting current year --- scripts/us_hud/income/process.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/scripts/us_hud/income/process.py b/scripts/us_hud/income/process.py index 22a4ad16af..2ac60953ee 100644 --- a/scripts/us_hud/income/process.py +++ b/scripts/us_hud/income/process.py @@ -69,7 +69,13 @@ def get_url(year): def download_with_retry(url): '''Retries downloading a file up to 5 times with exponential backoff.''' logging.info(f"Downloading URL: {url}") - return requests.get(url, verify=False) + headers = { + 'User-Agent': + 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36' + } + response = requests.get(url, headers=headers, timeout=60) + response.raise_for_status() + return response def download_file(url: str, filename: str, input_folder: str): @@ -140,7 +146,7 @@ def process_all(): if FLAGS.mode == "" or FLAGS.mode == "download": logging.info("Starting download phase...") - for year in range(2006, today.year): + for year in range(2006, today.year + 1): url = get_url(year) if url: filename = f"Section8-FY{year}.xlsx" if year > 2016 else f"Section8-FY{year}.xls" @@ -148,7 +154,7 @@ def process_all(): if FLAGS.mode == "" or FLAGS.mode == "process": logging.info("Starting processing phase...") - for year in range(2006, today.year): + for year in range(2006, today.year + 1): if not os.path.exists( os.path.join( input_folder, f"Section8-FY{year}.xlsx" From 926c5ad0793dbfba2d1010b99b73fb62ff1679d0 Mon Sep 17 00:00:00 2001 From: shvngisingh Date: Thu, 9 Jul 2026 15:14:29 +0530 Subject: [PATCH 2/2] Apply suggestion from @gemini-code-assist[bot] Co-authored-by: gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com> --- scripts/us_hud/income/process.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/scripts/us_hud/income/process.py b/scripts/us_hud/income/process.py index 2ac60953ee..74625a4941 100644 --- a/scripts/us_hud/income/process.py +++ b/scripts/us_hud/income/process.py @@ -150,6 +150,19 @@ def process_all(): url = get_url(year) if url: filename = f"Section8-FY{year}.xlsx" if year > 2016 else f"Section8-FY{year}.xls" + if year == today.year: + try: + headers = { + 'User-Agent': + 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36' + } + resp = requests.get(url, headers=headers, timeout=10, stream=True) + if resp.status_code != 200: + logging.warning(f"HUD income limits for {year} are not yet available. Skipping.") + continue + except Exception as e: + logging.warning(f"Could not check availability for {year}: {e}. Skipping.") + continue download_file(url, filename, input_folder) if FLAGS.mode == "" or FLAGS.mode == "process":