Upload folder using huggingface_hub

README.md

@@ -33,13 +33,13 @@ The model is trained on data from 2019-2022 and validated on data from 2022-2023
 
 ### Preprocessing
 
-Data is prepared with the `ocf_datapipes.training.pvnet` datapipe [2].
+Data is prepared with the `ocf_data_sampler/torch_datasets/datasets/pvnet_uk` Dataset [2].
 
 
 ## Results
 
 The training logs for the current model can be found here:
- - [https://wandb.ai/openclimatefix/PVNet_da/runs/6yy7obbu](https://wandb.ai/openclimatefix/PVNet_da/runs/6yy7obbu)
+ - [https://wandb.ai/openclimatefix/PVNet_ECMWF_DA/runs/r3rtnpq7](https://wandb.ai/openclimatefix/PVNet_ECMWF_DA/runs/r3rtnpq7)
 
 
 The training logs for all model runs of PVNet2 can be found [here](https://wandb.ai/openclimatefix/pvnet2.1).
@@ -54,4 +54,9 @@ Trained on a single NVIDIA Tesla T4
 ### Software
 
 - [1] https://github.com/openclimatefix/PVNet
-- [2] https://github.com/openclimatefix/ocf_datapipes
+- [2] https://github.com/openclimatefix/ocf-data-sampler
+
+
+The versions of these packages can be found below:
+ - ocf-data-sampler==0.2.17
+ - pvnet==4.1.2

config.json

@@ -10,15 +10,6 @@
         0.98
     ],
     "nwp_encoders_dict": {
-        "ukv": {
-            "_target_": "pvnet.models.multimodal.encoders.encoders3d.DefaultPVNet",
-            "_partial_": true,
-            "in_channels": 12,
-            "out_features": 256,
-            "number_of_conv3d_layers": 6,
-            "conv3d_channels": 32,
-            "image_size_pixels": 24
-        },
         "ecmwf": {
             "_target_": "pvnet.models.multimodal.encoders.encoders3d.DefaultPVNet",
             "_partial_": true,
@@ -29,7 +20,6 @@
             "image_size_pixels": 12
         }
     },
-    "add_image_embedding_channel": false,
     "pv_encoder": null,
     "output_network": {
         "_target_": "pvnet.models.multimodal.linear_networks.networks.ResFCNet2",
@@ -43,24 +33,20 @@
     "include_sun": true,
     "include_gsp_yield_history": false,
     "forecast_minutes": 2160,
-    "history_minutes": 120,
-    "pv_history_minutes": 180,
+    "history_minutes": 0,
     "nwp_history_minutes": {
-        "ukv": 120,
         "ecmwf": 120
     },
     "nwp_forecast_minutes": {
-        "ukv": 1800,
         "ecmwf": 2220
     },
     "nwp_interval_minutes": {
-        "ukv": 60,
         "ecmwf": 60
     },
     "optimizer": {
         "_target_": "pvnet.optimizers.EmbAdamWReduceLROnPlateau",
         "lr": 0.0001,
-        "weight_decay": 0.25,
+        "weight_decay": 0.01,
         "amsgrad": true,
         "patience": 5,
         "factor": 0.1,

data_config.yaml

@@ -4,14 +4,20 @@ general:
 input_data:
   gsp:
     dropout_fraction: 0
-    dropout_timedeltas_minutes: null
+    dropout_timedeltas_minutes: []
     forecast_minutes: 2160
-    history_minutes: 120
+    interval_end_minutes: 2160
+    interval_start_minutes: 0
     time_resolution_minutes: 30
-    gsp_zarr_path: PLACEHOLDER.zarr
+    zarr_path: PLACEHOLDER.zarr
   nwp:
     ecmwf:
-      nwp_channels:
+      accum_channels:
+      - dswrf
+      - dlwrf
+      - sr
+      - duvrs
+      channels:
       - t2m
       - dswrf
       - dlwrf
@@ -19,44 +25,84 @@ input_data:
       - mcc
       - lcc
       - tcc
-      - sde
+      - sd
       - sr
       - duvrs
       - u10
       - v10
       dropout_fraction: 1.0
       dropout_timedeltas_minutes:
-      - -180
-      nwp_image_size_pixels_height: 12
-      nwp_image_size_pixels_width: 12
-      forecast_minutes: 2220
-      history_minutes: 120
-      max_staleness_minutes: null
-      nwp_provider: ecmwf
-      time_resolution_minutes: 60
-      nwp_zarr_path: PLACEHOLDER.zarr
-    ukv:
-      nwp_channels:
-      - t
-      - dswrf
-      - dlwrf
-      - hcc
-      - mcc
-      - lcc
-      - sde
-      - r
-      - vis
-      - si10
-      - wdir10
-      - prate
-      dropout_fraction: 1.0
-      dropout_timedeltas_minutes:
-      - -180
-      nwp_image_size_pixels_height: 24
-      nwp_image_size_pixels_width: 24
-      forecast_minutes: 1800
-      history_minutes: 120
+      - -360
+      forecast_minutes: 2460.0
+      image_size_pixels_height: 12
+      image_size_pixels_width: 12
+      interval_end_minutes: 2220
+      interval_start_minutes: -120
       max_staleness_minutes: null
-      nwp_provider: ukv
+      normalisation_constants:
+        diff_dlwrf:
+          mean: 1136464.0
+          std: 131942.03125
+        diff_dswrf:
+          mean: 420584.6875
+          std: 715366.3125
+        diff_duvrs:
+          mean: 48265.4765625
+          std: 81605.25
+        diff_sr:
+          mean: 469169.5
+          std: 818950.6875
+        dlwrf:
+          mean: 27187026.0
+          std: 15855867.0
+        dswrf:
+          mean: 11458988.0
+          std: 13025427.0
+        duvrs:
+          mean: 1305651.25
+          std: 1445635.25
+        hcc:
+          mean: 0.3961029052734375
+          std: 0.42244860529899597
+        lcc:
+          mean: 0.44901806116104126
+          std: 0.3791404366493225
+        mcc:
+          mean: 0.3288780450820923
+          std: 0.38039860129356384
+        prate:
+          mean: 3.108070450252853e-05
+          std: 9.81039775069803e-05
+        sd:
+          mean: 8.107526082312688e-05
+          std: 0.000913831521756947
+        sr:
+          mean: 12905302.0
+          std: 16294988.0
+        t2m:
+          mean: 283.48333740234375
+          std: 3.692270040512085
+        tcc:
+          mean: 0.7049227356910706
+          std: 0.37487083673477173
+        u10:
+          mean: 1.7677178382873535
+          std: 5.531515598297119
+        u100:
+          mean: 2.393547296524048
+          std: 7.2320556640625
+        u200:
+          mean: 2.7963004112243652
+          std: 8.049470901489258
+        v10:
+          mean: 0.985887885093689
+          std: 5.411230564117432
+        v100:
+          mean: 1.4244288206100464
+          std: 6.944501876831055
+        v200:
+          mean: 1.6010299921035767
+          std: 7.561611652374268
+      provider: ecmwf
       time_resolution_minutes: 60
-      nwp_zarr_path: PLACEHOLDER.zarr
+      zarr_path: PLACEHOLDER.zarr

pytorch_model.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:3c64b8178010b5fa2b803c191addc79455b5b13142df4612aa00b6aea3bcb9d4
-size 91006822
+oid sha256:02d21a0d92ca182f3a7d0e1e0e0fd81789cf8b196dd9502d025dbaeb6067c1fc
+size 12284671