# Copyright 2013 – present by the SalishSeaCast Project contributors
# and The University of British Columbia
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# https://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# SPDX-License-Identifier: Apache-2.0
"""SalishSeaCast worker that downloads a daily averaged file from the
University of Washington Live Ocean model forecast product for a specified date.
The file contains a hyperslab that covers the SalishSeaCast NEMO model western
(Juan de Fuca) open boundary that is generated by UW running our
salishsea_tools.UBC_subdomain module.
"""
import logging
import os
import socket
from pathlib import Path
import arrow
import nemo_cmd.api
import requests
from dateutil import tz
from nemo_nowcast import NowcastWorker, WorkerError, get_web_data
from retrying import retry, RetryError
from nowcast import lib
NAME = "download_live_ocean"
logger = logging.getLogger(NAME)
[docs]
def main():
"""Set up and run the worker.
For command-line usage see:
:command:`python -m nowcast.workers.download_live_ocean -h`
"""
worker = NowcastWorker(NAME, description=__doc__)
worker.init_cli()
worker.cli.add_date_option(
"--run-date",
default=arrow.now().floor("day"),
help="Date to download the Live Ocean forecast product for.",
)
worker.run(download_live_ocean, success, failure)
return worker
def success(parsed_args):
ymd = parsed_args.run_date.format("YYYY-MM-DD")
logger.info(f"{ymd} Live Ocean file for Salish Sea western boundary downloaded")
msg_type = "success"
return msg_type
def failure(parsed_args):
ymd = parsed_args.run_date.format("YYYY-MM-DD")
logger.critical(
f"{ymd} Live Ocean file for Salish Sea western boundary download failed"
)
msg_type = "failure"
return msg_type
def download_live_ocean(parsed_args, config, *args):
yyyymmdd = parsed_args.run_date.format("YYYYMMDD")
dotted_yyyymmdd = parsed_args.run_date.format("YYYY.MM.DD")
ymd = parsed_args.run_date.format("YYYY-MM-DD")
logger.info(
f"downloading Salish Sea western boundary day-averaged LiveOcean file for {ymd}"
)
ts_config = config["temperature salinity"]
process_status_url_tmpl = ts_config["download"]["status file url template"]
process_status_url = process_status_url_tmpl.format(yyyymmdd=dotted_yyyymmdd)
with requests.Session() as session:
try:
_is_file_ready(process_status_url, session)
except RetryError as exc:
logger.error(
f"giving up after {exc.last_attempt.attempt_number} attempts: "
f"{exc.last_attempt.value[1]} for {process_status_url}"
)
raise WorkerError
bc_file_url_tmpl = ts_config["download"]["bc file url template"]
bc_file_url = bc_file_url_tmpl.format(yyyymmdd=dotted_yyyymmdd)
dest_dir = Path(ts_config["download"]["dest dir"], yyyymmdd)
filename = ts_config["download"]["file name"]
grp_name = config["file group"]
lib.mkdir(dest_dir, logger, grp_name=grp_name)
get_web_data(
bc_file_url, logger_name=NAME, filepath=dest_dir / filename, session=session
)
size = os.stat(dest_dir / filename).st_size
logger.info(
f"downloaded {size} bytes from {bc_file_url} to {dest_dir / filename}"
)
if size == 0:
logger.critical(f"Problem! 0 size file: {dest_dir / filename}")
raise WorkerError
nemo_cmd.api.deflate([dest_dir / filename], 1)
checklist = {ymd: os.fspath(dest_dir / filename)}
return checklist
def _retry_if_not_ready(ready):
return not ready
@retry(
retry_on_result=_retry_if_not_ready,
wait_fixed=5 * 60 * 1000,
stop_max_delay=3 * 3600 * 1000,
wrap_exception=True,
)
def _is_file_ready(process_status_url, session):
try:
response = session.get(process_status_url)
response.raise_for_status()
except (
requests.exceptions.ConnectionError,
requests.exceptions.HTTPError,
socket.error,
) as exc:
logger.debug(f"received {exc} from {process_status_url}")
raise exc
ready_time = arrow.get(response.text)
ready_time = arrow.get(ready_time.datetime, tz.gettz("US/Pacific"))
if ready_time < arrow.now(tz.gettz("US/Pacific")):
logger.debug(f"LiveOcean ubc.nc file generation completed at {ready_time}")
return True
else:
logger.debug("LiveOcean ubc.nc file processing not completed yet")
return False
if __name__ == "__main__":
main() # pragma: no cover