diff --git a/fast-magephem-service/.gitignore b/fast-magephem-service/.gitignore index 63bece1..87d46e0 100644 --- a/fast-magephem-service/.gitignore +++ b/fast-magephem-service/.gitignore @@ -5,9 +5,9 @@ core.* *~ *.csv *.err -*.so *.o +python/irene/c/*.so python/irene/c/common_include python/irene/c/lib python/irene/c/models_include -python/irene/lib +.dockerconfigjson \ No newline at end of file diff --git a/fast-magephem-service/Dockerfile b/fast-magephem-service/Dockerfile index 3fafb28..ca08204 100644 --- a/fast-magephem-service/Dockerfile +++ b/fast-magephem-service/Dockerfile @@ -1,24 +1,36 @@ -#FROM e3-devops.aero.org/ssdhas/python:3.9.13-buster -FROM python:3.9.13-buster +FROM python:3.9.17-slim-bookworm WORKDIR /app +#Removing warning in Jenkins about apt-util not being installed, causing containerization to hang +ENV DEBIAN_FRONTEND noninteractive +ENV DEBCONF_NOWARNINGS="yes" + +RUN apt-get update && apt-get install -y\ + build-essential \ + cmake \ + gfortran \ + libboost-all-dev \ + libopenmpi-dev \ + googletest \ + libhdf5-dev \ + libxerces-c-dev \ + && rm -rf /var/lib/apt/lists/* + +COPY requirements.txt . + +RUN pip install --no-cache-dir --upgrade pip setuptools \ + && pip install --no-cache-dir -r requirements.txt \ + && pip uninstall -y Flask \ + && pip install Flask==2.2.3 + COPY python /app/python COPY python/lib /usr/local/lib64/ COPY test /app/test -COPY requirements.txt . +RUN ln -s /usr/lib/x86_64-linux-gnu/hdf5/serial/libhdf5.so /usr/lib/x86_64-linux-gnu/libhdf5.so.103 -#RUN DOW=$(date +%a) \ -RUN apt-get update \ - && apt-get install -y build-essential \ - && apt-get install -y libhdf5-dev \ - && apt-get install -y libxerces-c-dev \ - && pip3 install --upgrade pip setuptools \ - && pip3 install -r requirements.txt \ - && apt-get clean \ - && rm -rf /var/lib/apt/lists/* \ - && ln -s /usr/lib/x86_64-linux-gnu/hdf5/serial/libhdf5.so /usr/lib/x86_64-linux-gnu/libhdf5.so.103 +# Run pytest tests +RUN pytest /app/test -# Run app.py CMD ["python", "/app/python/app.py"] diff --git a/fast-magephem-service/python/app.py b/fast-magephem-service/python/app.py index ec4f544..62cdd16 100644 --- a/fast-magephem-service/python/app.py +++ b/fast-magephem-service/python/app.py @@ -11,8 +11,7 @@ update to python 3.7+ to get fromisoformat replace strptime call modify OpenAPI YAML file to note time format S(.mmmuuu) instead of S.mmm(uuu) - add magephem service for OPQ - add magephem service for IGRF, T89 + add magephem service for T89 (when IRENE C++ code supports it) """ import sys diff --git a/fast-magephem-service/python/fast-magephem.yaml b/fast-magephem-service/python/fast-magephem.yaml index 077dfb7..1eeb865 100755 --- a/fast-magephem-service/python/fast-magephem.yaml +++ b/fast-magephem-service/python/fast-magephem.yaml @@ -30,6 +30,7 @@ paths: Supports GDZ, GEO, GSM, GSE, SM, GEI, MAG, SPH.
Provide input system (fromSys) and list of dates (dates), list of 3-D locations (xIN), and requested output system (outSys).
+ Supports up to 100,000 dates/locations
Returns same list of dates (dates) and list of 3-D locations (xOUT) in output system.
dates and xIN lists must have same length. xOUT will have that length as well.
- GDZ - geodetic as alt (km), latitude (deg), longitude (deg). @@ -67,6 +68,8 @@ paths: xIN: allOf: - $ref: '#/components/schemas/CoordinateList3D' + minItems: 0 + maxItems: 100000 description: "list of input 3-D coordinate sets in fromSys (nested list)" responses: '200': @@ -83,6 +86,8 @@ paths: xOUT: allOf: - $ref: '#/components/schemas/CoordinateList3D' + minItems: 0 + maxItems: 100000 description: "list of 3-D coordinate sets in toSys (nested list)" '405': description: Invalid input @@ -90,7 +95,7 @@ paths: post: tags: - magephem - summary: Translate from one 3-D frame to another + summary: Compute magnetic ephemeris description: | compute magnetic ephemeris / coordinates for a list of times and locations
@@ -112,6 +117,7 @@ paths: time x 3, or time x direciton x 3 - kext - external field model 'opq' or 'igrf' (=none) - outputs - list of coordinates to output (I,Bm,K,Phi,hmin,L,Blocal,MLT,Bmin,BminX,BminY,BminZ)
+ Supports up to 100,000 dates/locations, and up to 25 pitch angles/dirs
At most one of alpha, dirs is allowed. if neither alpha or dirs is provided, assumes alpha=90 @@ -147,6 +153,8 @@ paths: X: allOf: - $ref: '#/components/schemas/CoordinateList3D' + minItems: 0 + maxItems: 100000 description: "list of input 3-D coordinate sets (nested list)" sys: allOf: @@ -164,21 +172,37 @@ paths: description: scalar input pitch angle - type: array example: [10.0,45.0,90.0] + minItems: 0 + maxItems: 25 + description: "static list of input pitch angles" + items: + $ref: '#/components/schemas/PitchAngle' + - type: array + example: [[10.0,45.0,90.0],[20.0,55.0,95.0]] + minItems: 0 + maxItems: 100000 + description: "time-varying list of lists input pitch angles" items: - oneOf: - - $ref: '#/components/schemas/PitchAngle' - - type: array - items: - $ref: '#/components/schemas/PitchAngle' - description: "list of input pitch angles" + type: array + minItems: 0 + maxItems: 25 + items: + $ref: '#/components/schemas/PitchAngle' dirs: oneOf: - allOf: - $ref: '#/components/schemas/CoordinateList3D' - description: "list of input 3-D coordinate sets (nested list)" + minItems: 0 + maxItems: 25 + description: "list of static input 3-D coordinate sets (nested list)" - type: array + minItems: 0 + maxItems: 100000 items: - $ref: '#/components/schemas/CoordinateList3D' + allOf: + - $ref: '#/components/schemas/CoordinateList3D' + minItems: 0 + maxItems: 25 description: "list of lists of input 3-D coordinate sets (nested nested list)" outputs: type: array @@ -286,6 +310,8 @@ components: DateList: # supply "description" via {allOf : [ $ref : #/components/schemas/DateList],description:...} type: array + minItems: 0 + maxItems: 100000 items: type: string format: date-time @@ -296,10 +322,10 @@ components: example: [[ 100.0, 45.0, -73.0],[ 100.1, 45.2, -73.5]] items: type: array + minItems: 3 + maxItems: 3 items: format: number - minLength: 3 - maxLength: 3 example: 100.0 PitchAngle: type: number @@ -317,4 +343,4 @@ components: type: number default: -1.0E+30 - + \ No newline at end of file diff --git a/fast-magephem-service/python/irene/fast_invariants.py b/fast-magephem-service/python/irene/fast_invariants.py index 80528c9..dcb352d 100644 --- a/fast-magephem-service/python/irene/fast_invariants.py +++ b/fast-magephem-service/python/irene/fast_invariants.py @@ -56,7 +56,7 @@ assert _slash == '/', "non-linux systems not supported yet" _flyin_path = os.path.dirname(os.path.abspath(__file__)) _NN_files = { # file names by [kext][coord] w/o the final .mat, also w/o _Kp# for t89 - 'opq':{'Phi':'fastPhi_net','hmin':'fast_him_net'}, + 'opq':{'Phi':'fastPhi_net','hmin':'fast_hmin_net'}, 'igrf':{'Phi':'fastPhi_net_igrf','hmin':'fast_hmin_net_igrf'}, } _NNs = {key:{} for key in _NN_files} # nets themselves diff --git a/fast-magephem-service/python/irene/lib.zip b/fast-magephem-service/python/irene/lib.zip deleted file mode 100644 index 8624616..0000000 Binary files a/fast-magephem-service/python/irene/lib.zip and /dev/null differ diff --git a/fast-magephem-service/python/irene/lib/libmag_util.so b/fast-magephem-service/python/irene/lib/libmag_util.so new file mode 100644 index 0000000..ab38179 Binary files /dev/null and b/fast-magephem-service/python/irene/lib/libmag_util.so differ diff --git a/fast-magephem-service/python/lib.zip b/fast-magephem-service/python/lib.zip deleted file mode 100644 index 8624616..0000000 Binary files a/fast-magephem-service/python/lib.zip and /dev/null differ diff --git a/fast-magephem-service/python/lib/libAbMatrix.so b/fast-magephem-service/python/lib/libAbMatrix.so new file mode 100644 index 0000000..3ff37c6 Binary files /dev/null and b/fast-magephem-service/python/lib/libAbMatrix.so differ diff --git a/fast-magephem-service/python/lib/libCTimeValue.so b/fast-magephem-service/python/lib/libCTimeValue.so new file mode 100644 index 0000000..1746c93 Binary files /dev/null and b/fast-magephem-service/python/lib/libCTimeValue.so differ diff --git a/fast-magephem-service/python/lib/libCoordXform.so b/fast-magephem-service/python/lib/libCoordXform.so new file mode 100644 index 0000000..a8771cf Binary files /dev/null and b/fast-magephem-service/python/lib/libCoordXform.so differ diff --git a/fast-magephem-service/python/lib/libFloatMath.so b/fast-magephem-service/python/lib/libFloatMath.so new file mode 100644 index 0000000..dd046fd Binary files /dev/null and b/fast-magephem-service/python/lib/libFloatMath.so differ diff --git a/fast-magephem-service/python/lib/libGeoSpaceTime.so b/fast-magephem-service/python/lib/libGeoSpaceTime.so new file mode 100644 index 0000000..a1535ac Binary files /dev/null and b/fast-magephem-service/python/lib/libGeoSpaceTime.so differ diff --git a/fast-magephem-service/python/lib/libThreading.so b/fast-magephem-service/python/lib/libThreading.so new file mode 100644 index 0000000..83e7735 Binary files /dev/null and b/fast-magephem-service/python/lib/libThreading.so differ diff --git a/fast-magephem-service/python/lib/libTimeConv_c.so b/fast-magephem-service/python/lib/libTimeConv_c.so new file mode 100644 index 0000000..4376002 Binary files /dev/null and b/fast-magephem-service/python/lib/libTimeConv_c.so differ diff --git a/fast-magephem-service/python/lib/libUnitTestOutput.so b/fast-magephem-service/python/lib/libUnitTestOutput.so new file mode 100644 index 0000000..df91d8e Binary files /dev/null and b/fast-magephem-service/python/lib/libUnitTestOutput.so differ diff --git a/fast-magephem-service/python/lib/libVectorOps.so b/fast-magephem-service/python/lib/libVectorOps.so new file mode 100644 index 0000000..64a2f55 Binary files /dev/null and b/fast-magephem-service/python/lib/libVectorOps.so differ diff --git a/fast-magephem-service/python/lib/libaccummodel.so b/fast-magephem-service/python/lib/libaccummodel.so new file mode 100644 index 0000000..7799085 Binary files /dev/null and b/fast-magephem-service/python/lib/libaccummodel.so differ diff --git a/fast-magephem-service/python/lib/libadiabat.so b/fast-magephem-service/python/lib/libadiabat.so new file mode 100644 index 0000000..6f000d4 Binary files /dev/null and b/fast-magephem-service/python/lib/libadiabat.so differ diff --git a/fast-magephem-service/python/lib/libadiabatmodel.so b/fast-magephem-service/python/lib/libadiabatmodel.so new file mode 100644 index 0000000..4eaf4f6 Binary files /dev/null and b/fast-magephem-service/python/lib/libadiabatmodel.so differ diff --git a/fast-magephem-service/python/lib/libae9ap9model.so b/fast-magephem-service/python/lib/libae9ap9model.so new file mode 100644 index 0000000..96c3ecd Binary files /dev/null and b/fast-magephem-service/python/lib/libae9ap9model.so differ diff --git a/fast-magephem-service/python/lib/libaggregmodel.so b/fast-magephem-service/python/lib/libaggregmodel.so new file mode 100644 index 0000000..6be227d Binary files /dev/null and b/fast-magephem-service/python/lib/libaggregmodel.so differ diff --git a/fast-magephem-service/python/lib/libapplication.so b/fast-magephem-service/python/lib/libapplication.so new file mode 100644 index 0000000..1da709c Binary files /dev/null and b/fast-magephem-service/python/lib/libapplication.so differ diff --git a/fast-magephem-service/python/lib/libcammice.so b/fast-magephem-service/python/lib/libcammice.so new file mode 100644 index 0000000..cecb768 Binary files /dev/null and b/fast-magephem-service/python/lib/libcammice.so differ diff --git a/fast-magephem-service/python/lib/libcmagfield.so b/fast-magephem-service/python/lib/libcmagfield.so new file mode 100644 index 0000000..beb7a7b Binary files /dev/null and b/fast-magephem-service/python/lib/libcmagfield.so differ diff --git a/fast-magephem-service/python/lib/libdatetimeutil.so b/fast-magephem-service/python/lib/libdatetimeutil.so new file mode 100644 index 0000000..40fcd94 Binary files /dev/null and b/fast-magephem-service/python/lib/libdatetimeutil.so differ diff --git a/fast-magephem-service/python/lib/libdosemodel.so b/fast-magephem-service/python/lib/libdosemodel.so new file mode 100644 index 0000000..d00024b Binary files /dev/null and b/fast-magephem-service/python/lib/libdosemodel.so differ diff --git a/fast-magephem-service/python/lib/libephemmodel.so b/fast-magephem-service/python/lib/libephemmodel.so new file mode 100644 index 0000000..07f8ac6 Binary files /dev/null and b/fast-magephem-service/python/lib/libephemmodel.so differ diff --git a/fast-magephem-service/python/lib/libfileio.so b/fast-magephem-service/python/lib/libfileio.so new file mode 100644 index 0000000..80c97ef Binary files /dev/null and b/fast-magephem-service/python/lib/libfileio.so differ diff --git a/fast-magephem-service/python/lib/libfio.so b/fast-magephem-service/python/lib/libfio.so new file mode 100644 index 0000000..4e69260 Binary files /dev/null and b/fast-magephem-service/python/lib/libfio.so differ diff --git a/fast-magephem-service/python/lib/libinputproc.so b/fast-magephem-service/python/lib/libinputproc.so new file mode 100644 index 0000000..3258b86 Binary files /dev/null and b/fast-magephem-service/python/lib/libinputproc.so differ diff --git a/fast-magephem-service/python/lib/libkernelmodel.so b/fast-magephem-service/python/lib/libkernelmodel.so new file mode 100644 index 0000000..0c97a0c Binary files /dev/null and b/fast-magephem-service/python/lib/libkernelmodel.so differ diff --git a/fast-magephem-service/python/lib/liblegacymodel.so b/fast-magephem-service/python/lib/liblegacymodel.so new file mode 100644 index 0000000..7395f5b Binary files /dev/null and b/fast-magephem-service/python/lib/liblegacymodel.so differ diff --git a/fast-magephem-service/python/lib/libmag_util.so b/fast-magephem-service/python/lib/libmag_util.so new file mode 100644 index 0000000..ab38179 Binary files /dev/null and b/fast-magephem-service/python/lib/libmag_util.so differ diff --git a/fast-magephem-service/python/lib/liborbitprop.so b/fast-magephem-service/python/lib/liborbitprop.so new file mode 100644 index 0000000..8150638 Binary files /dev/null and b/fast-magephem-service/python/lib/liborbitprop.so differ diff --git a/fast-magephem-service/python/lib/libradenv.so b/fast-magephem-service/python/lib/libradenv.so new file mode 100644 index 0000000..fb731b8 Binary files /dev/null and b/fast-magephem-service/python/lib/libradenv.so differ diff --git a/fast-magephem-service/python/lib/libshielddose2.so b/fast-magephem-service/python/lib/libshielddose2.so new file mode 100644 index 0000000..50aee63 Binary files /dev/null and b/fast-magephem-service/python/lib/libshielddose2.so differ diff --git a/fast-magephem-service/python/lib/libspwxcommon.so b/fast-magephem-service/python/lib/libspwxcommon.so new file mode 100644 index 0000000..e3441e9 Binary files /dev/null and b/fast-magephem-service/python/lib/libspwxcommon.so differ diff --git a/fast-magephem-service/python/lib/libtaskproc.so b/fast-magephem-service/python/lib/libtaskproc.so new file mode 100644 index 0000000..f1c8eb8 Binary files /dev/null and b/fast-magephem-service/python/lib/libtaskproc.so differ diff --git a/fast-magephem-service/requirements.txt b/fast-magephem-service/requirements.txt index a67ce9e..5284f2e 100644 --- a/fast-magephem-service/requirements.txt +++ b/fast-magephem-service/requirements.txt @@ -2,4 +2,6 @@ connexion[swagger-ui]==2.14.1 numpy==1.22.4 h5py==3.7.0 scipy==1.9.3 -flask==2.2.5 +pytest==7.4.0 +xlsxwriter==3.1.2 +matplotlib==3.7.1 \ No newline at end of file diff --git a/fast-magephem-service/test/ssdhas_shared/python/test_utils.py b/fast-magephem-service/test/ssdhas_shared/python/test_utils.py index 782d2c4..7fc3543 100644 --- a/fast-magephem-service/test/ssdhas_shared/python/test_utils.py +++ b/fast-magephem-service/test/ssdhas_shared/python/test_utils.py @@ -5,18 +5,19 @@ import numpy as np import datetime as dt +from pytest import approx def recursive_equals(a,b,path='/'): if isinstance(a,np.ndarray): assert isinstance(b,np.ndarray), '%s are not both numpy ndarrays ' % path assert len(a.dtype) == len(b.dtype), '%s have different dtype structure' % path if len(a.dtype) == 0: # simple array - assert np.array_equal(a,b,equal_nan=np.issubdtype(a.dtype,float)), 'arrays %s are not equal ' % path + assert np.all_close(a,b,equal_nan=np.issubdtype(a.dtype,float)), 'arrays %s are not equal ' % path return True else: # structured array assert a.dtype.names != b.dtype.names, 'structured arrays %s have different keys ' % path for key in a.dtype.names: - assert np.array_equal(a[key],b[key],equal_nan=np.issubdtype(a[key].dtype,float)),'arrays %s/%s are not equal' % (path,key) + assert np.all_close(a[key],b[key],equal_nan=np.issubdtype(a[key].dtype,float)),'arrays %s/%s are not equal' % (path,key) return True if isinstance(a,dict): @@ -42,9 +43,14 @@ def recursive_equals(a,b,path='/'): assert np.issubdtype(type(b),base),'%s are not both %s' % (path,base.__name__) assert np.isscalar(a), 'a is not scalar %s' % path assert np.isscalar(b), 'b is not scalar %s' % path - if base == float: - assert (np.isnan(a) == np.isnan(b)), 'Only one is NaN at %s' % (path) - assert a==b, 'a != b at %s' % path + if base == float: # special case, handling nan and approximately equal + if np.isnan(a): + assert np.isnan(b), 'a is nan, b is not at %s' % (path) + else: + assert not np.isnan(b), 'b is nan, a is not at %s' % (path) + assert a==approx(b), 'a != b at %s' % path + else: # all other classes require exact equals + assert a==b, 'a != b at %s' % path return True assert False, 'Unable to compare %s types %s and %s' % (path,type(a),type(b))