Commit 5f3160c2: fix(worker) : un dictionnaire de conf venant du client est passé à l'initial...

create_indicator.ipynb

-12+213

/examples/create_indicator.ipynb

View

"cells": [

{

"cell_type": "code",

"execution_count": null,

"execution_count": 1,

"metadata": {},

"outputs": [],

"source": [

},

{

"cell_type": "code",

"execution_count": null,

"execution_count": 2,

"metadata": {},

"outputs": [],

"source": [

},

{

"cell_type": "code",

"execution_count": null,

"execution_count": 3,

"metadata": {},

"outputs": [],

"source": [

},

{

"cell_type": "code",

"execution_count": null,

"execution_count": 4,

"metadata": {},

"outputs": [],

"outputs": [

{

"name": "stderr",

"output_type": "stream",

"text": [

"20/02/2024 17:38:00 - INFO - Config - Settings Imported\n",

"20/02/2024 17:38:00 - WARNING - The following variables are null: commun_path, project_dir, data_catalog_dir, data_output_dir, sig_data_path, project_db_schema\n",

"20/02/2024 17:38:01 - INFO - Utils - Connection Imported\n",

"20/02/2024 17:38:01 - INFO - Utils - Dataframe Imported\n",

"20/02/2024 17:38:01 - INFO - Utils - Geometry Imported\n",

"20/02/2024 17:38:01 - DEBUG - Could not import boto3, continuing with reduced functionality.\n",

"20/02/2024 17:38:01 - INFO - Utils - Raster Imported\n",

"20/02/2024 17:38:02 - DEBUG - Could not import boto3, continuing with reduced functionality.\n",

"20/02/2024 17:38:02 - DEBUG - GDAL_DATA found in environment.\n",

"20/02/2024 17:38:02 - DEBUG - PROJ_DATA found in environment.\n",

"20/02/2024 17:38:02 - INFO - GeoIndicator - Gee Imported\n",

"20/02/2024 17:38:02 - INFO - Utils - Interpolation Imported\n",

"20/02/2024 17:38:02 - INFO - GeoIndicator - Distribution Imported\n",

"20/02/2024 17:38:02 - INFO - GeoIndicator - Raster Imported\n",

"20/02/2024 17:38:02 - INFO - GeoIndicator - Calculation Imported\n"

]

}

],

"source": [

"%load_ext autoreload\n",

"%autoreload 2\n",

},

{

"cell_type": "code",

"execution_count": null,

"execution_count": 5,

"metadata": {},

"outputs": [],

"outputs": [

{

"data": {

"text/html": [

"\n",

" <style>\n",

" .geemap-dark {\n",

" --jp-widgets-color: white;\n",

" --jp-widgets-label-color: white;\n",

" --jp-ui-font-color1: white;\n",

" --jp-layout-color2: #454545;\n",

" background-color: #383838;\n",

" }\n",

" \n",

" .geemap-dark .jupyter-button {\n",

" --jp-layout-color3: #383838;\n",

" }\n",

" \n",

" .geemap-colab {\n",

" background-color: var(--colab-primary-surface-color, white);\n",

" }\n",

" \n",

" .geemap-colab .jupyter-button {\n",

" --jp-layout-color3: var(--colab-primary-surface-color, white);\n",

" }\n",

" </style>\n",

" "

],

"text/plain": [

"<IPython.core.display.HTML object>"

]

},

"metadata": {},

"output_type": "display_data"

},

{

"name": "stderr",

"output_type": "stream",

"text": [

"20/02/2024 17:38:02 - WARNING - La variable d'environnement SCHEDULER_IP doit être renseignée pour effectuer les traitements de manière distribuée\n",

"20/02/2024 17:38:02 - INFO - on applique cette ip par défaut : 172.20.12.13:9786\n",

"20/02/2024 17:38:02 - DEBUG - Using selector: EpollSelector\n"

]

},

{

"data": {

"text/plain": [

"{'tcp://172.20.10.112:40691': None}"

]

},

"execution_count": 5,

"metadata": {},

"output_type": "execute_result"

}

],

"source": [

"client = settings.getDaskClient()\n",

"config = settings.initializeBilboProject(dotenvPath='.dev_env')\n",

"client.run(settings.initializeWorkers, config)\n",

"\n",

"client.run(settings.initializeWorkers)\n",

"\n",

"config = settings.initializeBilboProject(dotenvPath='.dev_env')\n",

"config"

"\n"

]

},

{

"cell_type": "code",

"execution_count": null,

"execution_count": 6,

"metadata": {},

"outputs": [

{

"data": {

"text/html": [

"\n",

" <style>\n",

" .geemap-dark {\n",

" --jp-widgets-color: white;\n",

" --jp-widgets-label-color: white;\n",

" --jp-ui-font-color1: white;\n",

" --jp-layout-color2: #454545;\n",

" background-color: #383838;\n",

" }\n",

" \n",

" .geemap-dark .jupyter-button {\n",

" --jp-layout-color3: #383838;\n",

" }\n",

" \n",

" .geemap-colab {\n",

" background-color: var(--colab-primary-surface-color, white);\n",

" }\n",

" \n",

" .geemap-colab .jupyter-button {\n",

" --jp-layout-color3: var(--colab-primary-surface-color, white);\n",

" }\n",

" </style>\n",

" "

],

"text/plain": [

"<IPython.core.display.HTML object>"

]

},

"metadata": {},

"outputs": [],

"output_type": "display_data"

},

{

"name": "stderr",

"output_type": "stream",

"text": [

"20/02/2024 17:38:07 - DEBUG - open file: /media/commun/commun/Informatique/SIG/Application/Jupyterhub/projets/catalogFiles/DWH_Dimensions.yaml\n",

"20/02/2024 17:38:07 - INFO - step list : {steplist}\n",

"20/02/2024 17:38:07 - INFO - Initial offset : 0 , limit : 10000\n",

"20/02/2024 17:38:07 - INFO - Id Spatial qui seront calculés : ['0', 'C', 'F', 'G', 'B', 'D', 'A', 'E', '1', '2', '3', '98827', '98817', '98813', '98826', '98819', '98816', '98828', '98824', '98832', '98821', '98831', '98818', '98804', '98822', '98814', '98815', '98806', '98809', '98820', '98811', '98807', '98805', '98803', '98823', '98810', '98812', '98833', '98808', '98825', '98801', '98830', '98802', '98829']\n",

"20/02/2024 17:38:07 - INFO - individu: H3_6_NC\n",

"20/02/2024 17:38:07 - INFO - indicateur: KBA\n",

"20/02/2024 17:38:07 - INFO - le nom de la table de faits en base de donnée en sortie de traitement répond au pré-requis : faits_kba_h3_nc_6\n",

"20/02/2024 17:38:07 - INFO - nbchuncks: 300\n",

"20/02/2024 17:38:07 - DEBUG - open file: /home/hugo/projets/bilbo-packages/examples/../test_catalog_file/h3.yaml\n",

"20/02/2024 17:38:07 - INFO - {'sources': {'compte_entites': {'driver': 'sql', 'metadata': {}, 'args': {'uri': 'postgresql://{{env(\"DB_USER\")}}:{{env(\"DB_PWD\")}}@{{env(\"DB_HOST\")}}:{{env(\"DB_PORT\")}}/oeil_traitement', 'sql_expr': 'SELECT COUNT(*) as nb FROM bilbo.h3_nc_6;'}, 'description': 'Compter le nombre d’entités sans charger les géométries'}}}\n",

"20/02/2024 17:38:07 - DEBUG - open file: /home/hugo/projets/bilbo-packages/examples/tmp.yaml\n",

"20/02/2024 17:38:07 - INFO - bilbo.h3_nc_6 nblignes : 759\n",

"20/02/2024 17:38:07 - INFO - sql_pagination : order by hex_id limit 10000 offset 0\n",

"20/02/2024 17:38:07 - INFO - Settings - getPaths data_config_file ../test_catalog_file/config/ \n",

"20/02/2024 17:38:07 - DEBUG - Settings - getPaths - config {'commun_path': '/media/commun/commun/', 'project_dir': '/media/commun/commun/Informatique/SIG/Application/Jupyterhub/projets/stac/', 'data_catalog_dir': '../test_catalog_file/', 'data_output_dir': '../target/', 'sig_data_path': '/media/commun/commun/Informatique/SIG/Donnees/', 'project_db_schema': 'bilbo', 'data_config_file': '../test_catalog_file/config/', 'dimension_catalog_dir': '/media/commun/commun/Informatique/SIG/Application/Jupyterhub/projets/catalogFiles/'}\n",

"20/02/2024 17:38:07 - INFO - create_indicator: Etape 1\n",

"20/02/2024 17:38:07 - DEBUG - open file: /home/hugo/projets/bilbo-packages/examples/../test_catalog_file/h3.yaml\n",

"20/02/2024 17:38:07 - INFO - create_indicator: Etape 1 sans bbox\n",

"20/02/2024 17:38:07 - INFO - create_indicator: Etape 1 --> indicateurSpec.get('catalogUri') and indicateurSpec.get('dataName') is not None\n",

"20/02/2024 17:38:07 - INFO - create_indicator: Etape 1 --> sourceType : None\n",

"20/02/2024 17:38:07 - INFO - source Type OTHER : ex . VECTOR \n",

"20/02/2024 17:38:07 - DEBUG - open file: /home/hugo/projets/bilbo-packages/examples/../test_catalog_file/data_reference.yaml\n",

"20/02/2024 17:38:07 - INFO - Calculation ...\n",

"20/02/2024 17:38:07 - INFO - with Dask - metaModelList : ' ['hex_id', 'kba', 'geometry', 'id_split']\n",

"20/02/2024 17:38:07 - INFO - reading intake source sources:\n",

" h3_nc_6:\n",

" args:\n",

" geopandas_kwargs:\n",

" crs: 3163\n",

" geom_col: geometry\n",

" sql_expr: select hex_id, geometry from bilbo.h3_nc_6 order by hex_id limit\n",

" 10000 offset 0\n",

" table: bilbo.h3_nc_6\n",

" uri: postgresql://hroussaffa:mcot@172.20.12.13:5432/oeil_traitement\n",

" description: \"Maille H3 niveau 6 sur les terres emerg\\xE9es de NC\"\n",

" driver: intake_geopandas.geopandas.PostGISSource\n",

" metadata:\n",

" catalog_dir: /home/hugo/projets/bilbo-packages/examples/../test_catalog_file/\n",

"...\n"

]

},

{

"name": "stdout",

"output_type": "stream",

"text": [

"sources:\n",

" h3_nc_6:\n",

" args:\n",

" geopandas_kwargs:\n",

" crs: 3163\n",

" geom_col: geometry\n",

" sql_expr: select hex_id, geometry from bilbo.h3_nc_6\n",

" table: bilbo.h3_nc_6\n",

" uri: postgresql://hroussaffa:mcot@172.20.12.13:5432/oeil_traitement\n",

" description: \"Maille H3 niveau 6 sur les terres emerg\\xE9es de NC\"\n",

" driver: intake_geopandas.geopandas.PostGISSource\n",

" metadata:\n",

" catalog_dir: /home/hugo/projets/bilbo-packages/examples/../test_catalog_file/\n",

"\n"

]

},

{

"name": "stderr",

"output_type": "stream",

"text": [

"20/02/2024 17:38:07 - DEBUG - df: 759\n",

"20/02/2024 17:38:07 - DEBUG - metaModelList ['hex_id', 'kba', 'geometry', 'id_split']\n",

"20/02/2024 17:38:07 - DEBUG - Load data in memory (759, 2)\n",

"20/02/2024 17:38:07 - DEBUG - converting to dask with chunksize 300\n",

"20/02/2024 17:38:08 - DEBUG - PROJ_ERROR: proj_create: unrecognized format / unknown name\n",

"20/02/2024 17:38:08 - DEBUG - data : Dask GeoDataFrame Structure:\n",

" hex_id geometry\n",

"npartitions=300 \n",

"0 string geometry\n",

"3 ... ...\n",

"... ... ...\n",

"757 ... ...\n",

"758 ... ...\n",

"Dask Name: to_pyarrow_string, 2 graph layers\n",

"20/02/2024 17:38:08 - DEBUG - func : <function generateIndicateur_parallel_v2 at 0x7f58ac352ca0>\n",

"20/02/2024 17:38:08 - INFO - Etape 1 - Result: <class 'dask_geopandas.core.GeoDataFrame'>\n",

"20/02/2024 17:38:08 - INFO - create_indicator: Etape 2\n",

"20/02/2024 17:38:08 - INFO - indexListIndicator None\n",

"20/02/2024 17:38:08 - INFO - create_indicator: Etape 2 --> pas de indexListIndicator\n"

]

}

],

"source": [

"%%time\n",

"\n",

settings.py

-46+21

/oeilnc_config/settings.py

View

if null_variables:

logging.warning("The following variables are null: {}".format(", ".join(null_variables)))

def initializeWorkers(dotenvPath='.env'):

def initializeWorkers(config_dict: dict):

load_dotenv(dotenv_path=dotenvPath)

global commun_path

global data_catalog_dir

global project_dir

global pswd

global host

global port

global home

global db_traitement

global db_ref

global db_externe

commun_path = getenv("COMMUN_PATH")

project_dir = getenv("PROJECT_PATH")

data_catalog_dir = getenv("DATA_CATALOG_DIR")

data_output_dir = getenv("DATA_OUTPUT_DIR")

sig_data_path = getenv("SIG_DATA_PATH")

project_db_schema = getenv("PROJECT_DB_SCHEMA")

data_config_file = getenv("DATA_CONFIG_DIR")

dimension_catalog_dir = getenv("DIM_CATALOG_DIR")

user = getenv("DB_USER")

pswd = getenv("DB_PWD")

host = getenv("DB_HOST")

port = getenv("DB_PORT")

home = getenv("HOME_PATH")

db_traitement = getenv("DB_WORKSPACE")

db_ref = getenv("DB_REF")

db_externe = getenv("DB_EXT")

config_dict = {

"user": user,

"pswd": pswd,

"host": host,

"port": port,

"home": home,

"db_traitement": db_traitement,

"db_ref": db_ref,

"db_externe": db_externe,

"commun_path": commun_path,

"project_dir": project_dir,

"data_catalog_dir": data_catalog_dir,

"data_output_dir": data_output_dir,

"sig_data_path": sig_data_path,

"project_db_schema": project_db_schema,

"data_config_file": data_config_file,

"dimension_catalog_dir": dimension_catalog_dir

}

logging.debug(f'Workers Settings - getPaths - config {config_dict}')

user = config_dict.get("user")

pswd = config_dict.get("pswd")

host = config_dict.get("host")

port = config_dict.get("port")

home = config_dict.get("home")

db_traitement = config_dict.get("db_traitement")

db_ref = config_dict.get("db_ref")

db_externe = config_dict.get("db_externe")

commun_path = config_dict.get("commun_path")

project_dir = config_dict.get("project_dir")

data_catalog_dir = config_dict.get("data_catalog_dir")

data_output_dir = config_dict.get("data_output_dir")

sig_data_path = config_dict.get("sig_data_path")

project_db_schema = config_dict.get("project_db_schema")

data_config_file = config_dict.get("data_config_file")

dimension_catalog_dir = config_dict.get("dimension_catalog_dir")

return

H3_6_NC.yaml

-2+2

/test_catalog_file/config/H3_6_NC.yaml

View

dataName: h3_nc_6

indexRef: hex_id

nbchuncks: 300

limit: 100

offset: 600

limit: 10000

offset: 0

epsg: EPSG:3163

overlayHow: intersection



	examples
	create_indicator.ipynb
	oeilnc_config
	settings.py
	test_catalog_file/config
	H3_6_NC.yaml