[feat] add support for image embedding re-use

pyproject.toml

@@ -23,8 +23,8 @@ python = "~3.10"
 python-dotenv = "^1.0.1"
 rasterio = "^1.3.9"
 requests = "^2.31.0"
-lisa-on-cuda = "^1.0.4"
-samgis-core = "^1.0.7"
+samgis-core = "^1.1.2"
+lisa-on-cuda = "^1.1.1"
 
 [tool.poetry.group.aws_lambda]
 optional = true

samgis_lisa_on_cuda/io/coordinates_pixel_conversion.py

@@ -1,5 +1,5 @@
 """functions useful to convert to/from latitude-longitude coordinates to pixel image coordinates"""
-from samgis_core.utilities.type_hints import tuple_float, tuple_float_any
+from samgis_core.utilities.type_hints import TupleFloat, TupleFloatAny
 
 from samgis_lisa_on_cuda import app_logger
 from samgis_lisa_on_cuda.utilities.constants import TILE_SIZE, EARTH_EQUATORIAL_RADIUS
@@ -82,7 +82,7 @@ def get_latlng_to_pixel_coordinates(
     return point
 
 
-def _from4326_to3857(lat: float, lon: float) -> tuple_float or tuple_float_any:
+def _from4326_to3857(lat: float, lon: float) -> TupleFloat or TupleFloatAny:
     from math import radians, log, tan
 
     x_tile: float = radians(lon) * EARTH_EQUATORIAL_RADIUS

samgis_lisa_on_cuda/io/geo_helpers.py

@@ -2,11 +2,11 @@
 from affine import Affine
 from numpy import ndarray as np_ndarray
 
-from samgis_core.utilities.type_hints import list_float, tuple_float, dict_str_int
+from samgis_core.utilities.type_hints import ListFloat, TupleFloat, DictStrInt
 from samgis_lisa_on_cuda import app_logger
 
 
-def load_affine_transformation_from_matrix(matrix_source_coefficients: list_float) -> Affine:
+def load_affine_transformation_from_matrix(matrix_source_coefficients: ListFloat) -> Affine:
     """
     Wrapper for rasterio.Affine.from_gdal() method
 
@@ -32,7 +32,7 @@ def load_affine_transformation_from_matrix(matrix_source_coefficients: list_floa
         raise e
 
 
-def get_affine_transform_from_gdal(matrix_source_coefficients: list_float or tuple_float) -> Affine:
+def get_affine_transform_from_gdal(matrix_source_coefficients: ListFloat or TupleFloat) -> Affine:
     """wrapper for rasterio Affine from_gdal method
 
     Args:
@@ -44,7 +44,7 @@ def get_affine_transform_from_gdal(matrix_source_coefficients: list_float or tup
     return Affine.from_gdal(*matrix_source_coefficients)
 
 
-def get_vectorized_raster_as_geojson(mask: np_ndarray, transform: tuple_float) -> dict_str_int:
+def get_vectorized_raster_as_geojson(mask: np_ndarray, transform: TupleFloat) -> DictStrInt:
     """
         Get shapes and values of connected regions in a dataset or array
 

samgis_lisa_on_cuda/io/tms2geotiff.py

@@ -1,7 +1,7 @@
 import os
 
 from numpy import ndarray
-from samgis_core.utilities.type_hints import tuple_float
+from samgis_core.utilities.type_hints import TupleFloat
 from xyzservices import TileProvider
 
 from samgis_lisa_on_cuda import app_logger
@@ -70,7 +70,7 @@ def download_extent(w: float, s: float, e: float, n: float, zoom: int or str = z
         raise e_download_extent
 
 
-def crop_raster(w: float, s: float, e: float, n: float, raster: ndarray, raster_bbox: tuple_float,
+def crop_raster(w: float, s: float, e: float, n: float, raster: ndarray, raster_bbox: TupleFloat,
                 crs: str = OUTPUT_CRS_STRING, driver: str = DRIVER_RASTERIO_GTIFF) -> tuple_ndarray_transform:
     """
     Crop a raster using given bounding box (w, s, e, n) values
@@ -134,7 +134,7 @@ def crop_raster(w: float, s: float, e: float, n: float, raster: ndarray, raster_
         raise e_crop_raster
 
 
-def get_transform_raster(raster: ndarray, raster_bbox: tuple_float) -> tuple_ndarray_transform:
+def get_transform_raster(raster: ndarray, raster_bbox: TupleFloat) -> tuple_ndarray_transform:
     """
     Convert the input raster image to RGB and extract the Affine 
 

samgis_lisa_on_cuda/io/wrappers_helpers.py

@@ -238,3 +238,25 @@ def get_url_tile(source_type: str):
 
 def check_source_type_is_terrain(source: str | TileProvider):
     return isinstance(source, TileProvider) and source.name in list(XYZTerrainProvidersNames)
+
+
+def get_source_name(source: str | TileProvider) -> str | bool:
+    try:
+        match source.lower():
+            case XYZDefaultProvidersNames.DEFAULT_TILES_NAME_SHORT:
+                source_output = providers.query_name(XYZDefaultProvidersNames.DEFAULT_TILES_NAME)
+            case _:
+                source_output = providers.query_name(source)
+        if isinstance(source_output, str):
+            return source_output
+        try:
+            source_dict = dict(source_output)
+            app_logger.info(f"source_dict:{type(source_dict)}, {'name' in source_dict}, source_dict:{source_dict}.")
+            return source_dict["name"]
+        except KeyError as ke:
+            app_logger.error(f"ke:{ke}.")
+    except ValueError as ve:
+        app_logger.info(f"source name::{source}, ve:{ve}.")
+    app_logger.info(f"source name::{source}.")
+
+    return False

samgis_lisa_on_cuda/prediction_api/global_models.py

@@ -2,3 +2,5 @@ models_dict = {
     "fastsam": {"instance": None},
     "lisa": {"inference": None}
 }
+embedding_dict = {}
+

samgis_lisa_on_cuda/prediction_api/lisa.py

@@ -1,7 +1,7 @@
 from datetime import datetime
 
 from lisa_on_cuda.utils import app_helpers
-from samgis_core.utilities.type_hints import llist_float, dict_str_int
+from samgis_core.utilities.type_hints import LlistFloat, DictStrInt
 from samgis_lisa_on_cuda import app_logger
 from samgis_lisa_on_cuda.io.geo_helpers import get_vectorized_raster_as_geojson
 from samgis_lisa_on_cuda.io.raster_helpers import write_raster_png, write_raster_tiff
@@ -13,12 +13,13 @@ msg_write_tmp_on_disk = "found option to write images and geojson output..."
 
 
 def lisa_predict(
-        bbox: llist_float,
+        bbox: LlistFloat,
         prompt: str,
         zoom: float,
         inference_function_name_key: str = "lisa",
-        source: str = DEFAULT_URL_TILES
-) -> dict_str_int:
+        source: str = DEFAULT_URL_TILES,
+        source_name: str = None
+) -> DictStrInt:
     """
     Return predictions as a geojson from a geo-referenced image using the given input prompt.
 
@@ -33,6 +34,7 @@ def lisa_predict(
         zoom: Level of detail
         inference_function_name_key: machine learning model name
         source: xyz
+        source_name: name of tile provider
 
     Returns:
         Affine transform
@@ -54,9 +56,9 @@ def lisa_predict(
     app_logger.info(
         f"img type {type(img)} with shape/size:{img.size}, transform type: {type(transform)}, transform:{transform}.")
     folder_write_tmp_on_disk = getenv("WRITE_TMP_ON_DISK", "")
+    prefix = f"w{pt1[1]},s{pt1[0]},e{pt0[1]},n{pt0[0]}_"
     if bool(folder_write_tmp_on_disk):
         now = datetime.now().strftime('%Y%m%d_%H%M%S')
-        prefix = f"w{pt1[1]},s{pt1[0]},e{pt0[1]},n{pt0[0]}_"
         app_logger.info(msg_write_tmp_on_disk + f"with coords {prefix}, shape:{img.shape}, {len(img.shape)}.")
         if img.shape and len(img.shape) == 2:
             write_raster_tiff(img, transform, f"{prefix}_{now}_", f"raw_tiff", folder_write_tmp_on_disk)
@@ -65,7 +67,9 @@ def lisa_predict(
     else:
         app_logger.info("keep all temp data in memory...")
 
-    _, mask, output_string = inference_fn(prompt, img)
+    app_logger.info(f"source_name:{source_name}, source_name type:{type(source_name)}.")
+    embedding_key = f"{source_name}_z{zoom}_{prefix}"
+    _, mask, output_string = inference_fn(prompt, img, app_logger, embedding_key)
     # app_logger.info(f"created {n_predictions} masks, preparing conversion to geojson...")
     return {
         "output_string": output_string,

samgis_lisa_on_cuda/prediction_api/predictors.py

@@ -4,21 +4,21 @@ from samgis_lisa_on_cuda.io.geo_helpers import get_vectorized_raster_as_geojson
 from samgis_lisa_on_cuda.io.raster_helpers import get_raster_terrain_rgb_like, get_rgb_prediction_image
 from samgis_lisa_on_cuda.io.tms2geotiff import download_extent
 from samgis_lisa_on_cuda.io.wrappers_helpers import check_source_type_is_terrain
-from samgis_lisa_on_cuda.prediction_api.global_models import models_dict
+from samgis_lisa_on_cuda.prediction_api.global_models import models_dict, embedding_dict
 from samgis_lisa_on_cuda.utilities.constants import DEFAULT_URL_TILES, SLOPE_CELLSIZE
-from samgis_core.prediction_api.sam_onnx import SegmentAnythingONNX
-from samgis_core.prediction_api.sam_onnx import get_raster_inference
+from samgis_core.prediction_api.sam_onnx import SegmentAnythingONNX, get_raster_inference_with_embedding_from_dict
 from samgis_core.utilities.constants import MODEL_ENCODER_NAME, MODEL_DECODER_NAME, DEFAULT_INPUT_SHAPE
-from samgis_core.utilities.type_hints import llist_float, dict_str_int, list_dict
+from samgis_core.utilities.type_hints import LlistFloat, DictStrInt, ListDict
 
 
 def samexporter_predict(
-        bbox: llist_float,
-        prompt: list_dict,
+        bbox: LlistFloat,
+        prompt: ListDict,
         zoom: float,
         model_name_key: str = "fastsam",
-        source: str = DEFAULT_URL_TILES
-) -> dict_str_int:
+        source: str = DEFAULT_URL_TILES,
+        source_name: str = None
+) -> DictStrInt:
     """
     Return predictions as a geojson from a geo-referenced image using the given input prompt.
 
@@ -33,6 +33,7 @@ def samexporter_predict(
         zoom: Level of detail
         model_name_key: machine learning model name
         source: xyz
+        source_name: name of tile provider
 
     Returns:
         Affine transform
@@ -60,8 +61,10 @@ def samexporter_predict(
 
     app_logger.info(
         f"img type {type(img)} with shape/size:{img.size}, transform type: {type(transform)}, transform:{transform}.")
-
-    mask, n_predictions = get_raster_inference(img, prompt, models_instance, model_name_key)
+    app_logger.info(f"source_name:{source_name}, source_name type:{type(source_name)}.")
+    embedding_key = f"{source_name}_z{zoom}_w{pt1[1]},s{pt1[0]},e{pt0[1]},n{pt0[0]}"
+    mask, n_predictions = get_raster_inference_with_embedding_from_dict(
+        img, prompt, models_instance, model_name_key, embedding_key, embedding_dict)
     app_logger.info(f"created {n_predictions} masks, preparing conversion to geojson...")
     return {
         "n_predictions": n_predictions,

wrappers/fastapi_wrapper.py

@@ -88,7 +88,7 @@ def post_test_string(request_input: StringPromptApiRequestBody) -> JSONResponse:
 @app.post("/infer_lisa")
 def infer_lisa(request_input: StringPromptApiRequestBody) -> JSONResponse:
     from samgis_lisa_on_cuda.prediction_api import lisa
-    from samgis_lisa_on_cuda.io.wrappers_helpers import get_parsed_bbox_points_with_string_prompt
+    from samgis_lisa_on_cuda.io.wrappers_helpers import get_parsed_bbox_points_with_string_prompt, get_source_name
 
     app_logger.info("starting lisa inference request...")
 
@@ -100,9 +100,11 @@ def infer_lisa(request_input: StringPromptApiRequestBody) -> JSONResponse:
         app_logger.info(f"lisa body_request:{body_request}.")
         app_logger.info(f"lisa module:{lisa}.")
         try:
+            source_name = get_source_name(request_input.source_type)
+            app_logger.info(f"source_name = {source_name}.")
             output = lisa.lisa_predict(
                 bbox=body_request["bbox"], prompt=body_request["prompt"], zoom=body_request["zoom"],
-                source=body_request["source"]
+                source=body_request["source"], source_name=source_name
             )
             duration_run = time.time() - time_start_run
             app_logger.info(f"duration_run:{duration_run}.")
@@ -132,7 +134,7 @@ def infer_lisa(request_input: StringPromptApiRequestBody) -> JSONResponse:
 @app.post("/infer_samgis")
 def infer_samgis(request_input: ApiRequestBody) -> JSONResponse:
     from samgis_lisa_on_cuda.prediction_api import predictors
-    from samgis_lisa_on_cuda.io.wrappers_helpers import get_parsed_bbox_points_with_dictlist_prompt
+    from samgis_lisa_on_cuda.io.wrappers_helpers import get_parsed_bbox_points_with_dictlist_prompt, get_source_name
 
     app_logger.info("starting plain samgis inference request...")
 
@@ -143,9 +145,11 @@ def infer_samgis(request_input: ApiRequestBody) -> JSONResponse:
         body_request = get_parsed_bbox_points_with_dictlist_prompt(request_input)
         app_logger.info(f"body_request:{body_request}.")
         try:
+            source_name = get_source_name(request_input.source_type)
+            app_logger.info(f"source_name = {source_name}.")
             output = predictors.samexporter_predict(
                 bbox=body_request["bbox"], prompt=body_request["prompt"], zoom=body_request["zoom"],
-                source=body_request["source"]
+                source=body_request["source"], source_name=source_name
             )
             duration_run = time.time() - time_start_run
             app_logger.info(f"duration_run:{duration_run}.")
@@ -208,7 +212,7 @@ async def lisa() -> FileResponse:
     return FileResponse(path=WORKDIR / "static" / "dist" / "lisa.html", media_type="text/html")
 
 
-app.mount("/", StaticFiles(directory=WORKDIR / "static" / "dist", html=True), name="static")
+app.mount("/", StaticFiles(directory=WORKDIR / "static" / "dist", html=True), name="root")
 
 
 @app.get("/")