Skip to content

Commit

Permalink
Add files via upload
Browse files Browse the repository at this point in the history
  • Loading branch information
@SebChami
SebChami authored Feb 11, 2025
1 parent 601627d commit 8fd4879
Show file tree
Hide file tree
Showing 5 changed files with 98 additions and 11 deletions.
5 changes: 2 additions & 3 deletions 5.3_Validation/pgm/SETENV-XML-EG.sas
View file
Original file line number Diff line number Diff line change
@@ -1,7 +1,5 @@
/*specify the path to the folder where the software has been dezipped : */
%LET eusilc=___PATH___ ;


*%LET eusilc=___PATH___ ;

%LET _dirsp_=/;
OPTIONS NOSOURCE MCOMPILENOTE=ALL;
Expand Down Expand Up @@ -45,6 +43,7 @@ LIBNAME BACK "&eusilc&_dirsp_.main&_dirsp_%LOWCASE(&cc)&_dirsp_.tmp";

/** LIBRARY LIST (2/2)*/
LIBNAME CSV "&eusilc&_dirsp_.main&_dirsp_%LOWCASE(&cc)&_dirsp_.csv";
LIBNAME RAW_DB "&eusilc&_dirsp_%quote(main)&_dirsp_%LOWCASE(&CC)&_dirsp_%LOWCASE(&SS&YY)" compress=yes;
LIBNAME XMLCFG "&eusilc&_dirsp_.XML_CONFIG&_dirsp_.XML";
LIBNAME MAPDIR "&eusilc&_dirsp_.XML_CONFIG&_dirsp_.MAP";

Expand Down
5 changes: 2 additions & 3 deletions 5.3_Validation/pgm/SETENV-XML-SA.sas
View file
Original file line number Diff line number Diff line change
@@ -1,15 +1,13 @@
/*specify the the path to the folder where the software has been dezipped : */
%LET eusilc=___PATH___ ;



/* / for UNIX environments, \ for windows environments */
%LET _dirsp_=/;
OPTIONS NOSOURCE MCOMPILENOTE=ALL;

/*** Parameters to specify */
%LET CC= ; /*Country bigram in uppercase letters*/
%LET YYYY=2024; /*Year (4 digits)*/
%LET YYYY=2023; /*Year (4 digits)*/
%LET EXTENSION=pdf; /*pdf by default, could be changed to docx (only if sas>=9.4, keep pdf otherwise) */
%LET CSV=NO; /*NO by default, could be changed to YES to obtain detailed list of errors in csv files */

Expand Down Expand Up @@ -50,6 +48,7 @@ LIBNAME BACK "&eusilc&_dirsp_.main&_dirsp_%LOWCASE(&cc)&_dirsp_.tmp";

/** LIBRARY LIST (2/2) */
LIBNAME CSV "&eusilc&_dirsp_.main&_dirsp_%LOWCASE(&cc)&_dirsp_.csv";
LIBNAME RAW_DB "&eusilc&_dirsp_%quote(main)&_dirsp_%LOWCASE(&CC)&_dirsp_%LOWCASE(&SS&YY)" compress=yes;
LIBNAME XMLCFG "&eusilc&_dirsp_.XML_CONFIG&_dirsp_.XML";
LIBNAME MAPDIR "&eusilc&_dirsp_.XML_CONFIG&_dirsp_.MAP";

Expand Down
79 changes: 79 additions & 0 deletions 5.3_Validation/pgm/SETENV-XML-VIYA.sas
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,79 @@
/* Configuration requirements for S3 access
THE SCRIPT WORKS PROPERLY ONLY IF SAS STUDIO IS STARTED USING SILC Context
The parameters are defined at the level of SILC Context.*/
%let parm_s3_region=lux;
%let parm_s3_authd=SILC_S3_Auth;
%let parm_s3_credentials=P01;
%let gRootPathS3=/silc-dev-01;


/*specify the path to the folder where the software has been dezipped : */
*%LET eusilc=___PATH___ ;

%LET _dirsp_=/;
OPTIONS NOSOURCE MCOMPILENOTE=ALL;

%LET yy=%SUBSTR(&yyyy,3,2);


/** NB OF ROTATION YEARS FOR PANEL */
%LET ROTATION=4;
%IF (&CC=BE or &CC=BG or &CC=IT or &CC=YY) %THEN %DO;%LET ROTATION=6;%END;
%IF (&CC=IE or &CC=NL or &CC=SE) AND &YY>21 %THEN %DO;%LET ROTATION=6;%END;
%IF (&CC=IE or &CC=NL or &CC=SE) AND &YY=21 %THEN %DO;%LET ROTATION=5;%END;


/*option to enable directory creation automaticaly*/
options dlcreatedir;
/*create parent folders before assigning child libraries below, librairies are not kept*/
LIBNAME silc "&eusilc";LIBNAME silc clear;
LIBNAME main "&eusilc&_dirsp_.main";LIBNAME main clear;
LIBNAME cc "&eusilc&_dirsp_.main&_dirsp_%lowcase(&cc)";LIBNAME cc clear;
LIBNAME XML_C "&eusilc&_dirsp_.XML_CONFIG";LIBNAME XML_C clear;


/** LIBRARY LIST (1/2)*/
LIBNAME RAW "&eusilc&_dirsp_.main&_dirsp_%lowcase(&cc)&_dirsp_.stage";
LIBNAME OUT "&eusilc&_dirsp_.main&_dirsp_%lowcase(&cc)&_dirsp_.out";
LIBNAME BACK "&eusilc&_dirsp_.main&_dirsp_%lowcase(&cc)&_dirsp_.tmp";
LIBNAME CSV "&eusilc&_dirsp_.main&_dirsp_%lowcase(&cc)&_dirsp_.csv";
LIBNAME RAW_DB "&eusilc&_dirsp_%quote(main)&_dirsp_%LOWCASE(&CC)&_dirsp_%LOWCASE(&SS&YY)" compress=yes;

/** LIBRARY LIST (2/2)*/
LIBNAME XMLCFG "&eusilc&_dirsp_.XML_CONFIG&_dirsp_.XML";
LIBNAME MAPDIR "&eusilc&_dirsp_.XML_CONFIG&_dirsp_.MAP";


/*S3 DOWNLOAD*/
/*create folders to download XML files*/
/*create parent folders before assigning child libraries below, librairies are not kept*/
LIBNAME XMLSVAL "&eusilc&_dirsp_.XML_CONFIG&_dirsp_.XML&_dirsp_.SVAL";LIBNAME XMLSVAL clear;
LIBNAME XMLLVAL "&eusilc&_dirsp_.XML_CONFIG&_dirsp_.XML&_dirsp_.SVAL";LIBNAME XMLLVAL clear;
LIBNAME XMLSCL "&eusilc&_dirsp_.XML_CONFIG&_dirsp_.XML&_dirsp_.SVAL";LIBNAME XMLSCL clear;
LIBNAME XMLCOMP "&eusilc&_dirsp_.XML_CONFIG&_dirsp_.XML&_dirsp_.SVAL";LIBNAME XMLCOMP clear;
LIBNAME XMLSIZE "&eusilc&_dirsp_.XML_CONFIG&_dirsp_.XML&_dirsp_.SVAL";LIBNAME XMLSIZE clear;
LIBNAME XMLOUTL "&eusilc&_dirsp_.XML_CONFIG&_dirsp_.XML&_dirsp_.SVAL";LIBNAME XMLOUTL clear;

/*download file by folder*/
proc s3 REGION="lux" AUTHDOMAIN="SILC_S3_Auth" CREDENTIALSPROFILE="P01";
/*csv files*/
getdir "&gRootPathS3/main&_dirsp_%lowcase(&cc)&_dirsp_.csv" "&eusilc/main&_dirsp_%lowcase(&cc)";

/*xml files and map files*/
getdir "&gRootPathS3&_dirsp_.XML_CONFIG&_dirsp_.MAP" "&eusilc&_dirsp_.XML_CONFIG";
getdir "&gRootPathS3&_dirsp_.XML_CONFIG&_dirsp_.XML&_dirsp_.SVAL" "&eusilc&_dirsp_.XML_CONFIG&_dirsp_.XML";
getdir "&gRootPathS3&_dirsp_.XML_CONFIG&_dirsp_.XML&_dirsp_.LVAL" "&eusilc&_dirsp_.XML_CONFIG&_dirsp_.XML";
getdir "&gRootPathS3&_dirsp_.XML_CONFIG&_dirsp_.XML&_dirsp_.SCL" "&eusilc&_dirsp_.XML_CONFIG&_dirsp_.XML" ;
getdir "&gRootPathS3&_dirsp_.XML_CONFIG&_dirsp_.XML&_dirsp_.COMP" "&eusilc&_dirsp_.XML_CONFIG&_dirsp_.XML";
getdir "&gRootPathS3&_dirsp_.XML_CONFIG&_dirsp_.XML&_dirsp_.SIZE" "&eusilc&_dirsp_.XML_CONFIG&_dirsp_.XML";
getdir "&gRootPathS3&_dirsp_.XML_CONFIG&_dirsp_.XML&_dirsp_.OUTL" "&eusilc&_dirsp_.XML_CONFIG&_dirsp_.XML";
run;


/** EXECUTION MODE */
%LET _IDX_DEBUG_=1;
%PUT _USER_;




14 changes: 13 additions & 1 deletion 5.3_Validation/pgm/WORKFLOW.sas
View file
Original file line number Diff line number Diff line change
Expand Up @@ -261,14 +261,24 @@ OPTIONS FmtErr;*NOSOURCE NOMPRINT NOMLOGIC;


%PUT *I* WRITING SILC RAW DATASETS ... ;
LIBNAME RAW_DB "&eusilc&_dirsp_%quote(main)&_dirsp_%LOWCASE(&CC)&_dirsp_%LOWCASE(&SS&YY)" compress=yes;
DATA RAW_DB.&ss&cc&YY.D;SET RAW.&ss&cc&YY.D;RUN;
DATA RAW_DB.&ss&cc&YY.R;SET RAW.&ss&cc&YY.R;RUN;
DATA RAW_DB.&ss&cc&YY.H;SET RAW.&ss&cc&YY.H;RUN;
DATA RAW_DB.&ss&cc&YY.P;SET RAW.&ss&cc&YY.P;RUN;

%END;

/*test if we are in Viya context*/;
%IF "%SUBSTR(&EUSILC,1,14)"="/opt/sas/viya/" %THEN %DO;
/*export results to S3*/
proc s3 REGION="lux" AUTHDOMAIN="SILC_S3_Auth" CREDENTIALSPROFILE="P01";
/*datasets*/
putdir "&eusilc&_dirsp_.main&_dirsp_%LOWCASE(&CC)&_dirsp_%LOWCASE(&SS&YY)" "&gRootPathS3&_dirsp_.main&_dirsp_%LOWCASE(&CC)&_dirsp_%LOWCASE(&SS&YY)";
/*pdf/docx and csv results*/
putdir "&eusilc&_dirsp_.main&_dirsp_%lowcase(&cc)&_dirsp_.out" "&gRootPathS3&_dirsp_.main&_dirsp_%lowcase(&cc)&_dirsp_.out";
run;
%END;

%MEND VALID_MAIN;

%MACRO clean_DS(F=P);
Expand Down Expand Up @@ -297,3 +307,5 @@ QUIT;

%VALID_INCLUDE;
%VALID_MAIN;


6 changes: 2 additions & 4 deletions 5.3_Validation/pgm/XML_SVAL.sas
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1746,10 +1746,8 @@ QUIT;
;
LENGTH VARS $32767;
RETAIN VARS '';
IF TYPE NOT IN ( 'HGRID' ) THEN DO;
VARS = TRIM(VARS) || TRIM(VARIABLE) || ':' || SUBSTR(FORMAT,1,1);
IF NOT EOF THEN VARS = TRIM(VARS) || '|';
END;
VARS = TRIM(VARS) || TRIM(VARIABLE) || ':' || SUBSTR(FORMAT,1,1);
IF NOT EOF THEN VARS = TRIM(VARS) || '|';
IF EOF THEN CALL SYMPUT('VARS',TRIM(VARS));
RUN;

Expand Down

0 comments on commit 8fd4879

Please sign in to comment.