Add: build, deploy, unpack, & run scripts...

Delete oldest file once we have ${directory.storage.maxcount} files in storage
Clean up large temp file in finally block

.gitignore

@@ -3,3 +3,4 @@
 .settings
 .vscode
 target/
+lib/

README.md

@@ -1 +1,11 @@
-# dods-harvester
+# dods-harvester
+
+#### Build, deploy, and run
+```bash
+./build.sh && ./push-deploy.sh
+```
+Then `ssh` to `maracoos-tds-harvest` and
+```bash
+cd /opt/dods-harvester
+./unpack-deploy.sh && ./run-harvester.sh
+```

build.sh

@@ -0,0 +1,4 @@
+#!/bin/bash
+mvn clean
+mvn install dependency:copy-dependencies
+mvn package

pom.xml

@@ -6,10 +6,11 @@
 	<modelVersion>4.0.0</modelVersion>
 
 	<groupId>com.rpsgroup</groupId>
-	<artifactId>lis_fvcom_harvest</artifactId>
+	<artifactId>dods-harvester</artifactId>
 	<version>1.0-SNAPSHOT</version>
+	<packaging>jar</packaging>
 
-	<name>lis_fvcom_harvest</name>
+	<name>dods-harvester</name>
 
 	<properties>
 		<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
@@ -82,10 +83,6 @@
 					<artifactId>maven-surefire-plugin</artifactId>
 					<version>2.22.1</version>
 				</plugin>
-				<plugin>
-					<artifactId>maven-jar-plugin</artifactId>
-					<version>3.0.2</version>
-				</plugin>
 				<plugin>
 					<artifactId>maven-install-plugin</artifactId>
 					<version>2.5.2</version>

push-deploy.sh

@@ -0,0 +1,6 @@
+#!/bin/bash
+mv target/dods-harvester*.jar target/dependency
+cd target/dependency
+tar -czf dods-harvester.tar.gz ./*
+scp dods-harvester.tar.gz maracoos-tds-harvest:/opt/dods-harvester/
+rm -f dods-harvester.tar.gz

run-harvester.sh

@@ -0,0 +1,3 @@
+#!/bin/bash
+java -cp ".:lib/*" com.rpsgroup.HarvesterMain
+

src/main/java/com/rpsgroup/FileManager.java

@@ -9,19 +9,21 @@ public class FileManager {
 
 	private PropertiesHolder props;
 
+	private FilenameFilter filter = new FilenameFilter() {
+
+		@Override
+		public boolean accept(File dir, String name) {
+			if (name.contains(props.ncFilename)) {
+				return true;
+			}
+			return false;
+		}
+	};
+
 	public Date getMostRecentDataTime() {
 
 		File dataDir = new File(props.storageDirectory);	
-		File[] dataFiles = dataDir.listFiles(new FilenameFilter() {
-
-			@Override
-			public boolean accept(File dir, String name) {
-				if (name.contains(props.ncFilename)) {
-					return true;
-				}
-				return false;
-			}
-		});
+		File[] dataFiles = dataDir.listFiles(filter);
 
 		long max = 0L;
 		for (int n=0; n<dataFiles.length; n++) {
@@ -36,6 +38,33 @@ public boolean accept(File dir, String name) {
 		return new Date(max);
 	}
 
+	public File getOldestFile() {
+
+		File dataDir = new File(props.storageDirectory);	
+		File[] dataFiles = dataDir.listFiles(filter);
+
+		long min = Long.MAX_VALUE;
+		File oldest = null;
+
+		for (int n=0; n<dataFiles.length; n++) {
+
+			File f = dataFiles[n];
+			long lastMod = f.lastModified();
+
+			if (lastMod < min) {
+				min = lastMod;
+				oldest = f;
+			}
+		}
+		return oldest;
+	}
+
+	public int getFileCount() {
+		File dataDir = new File(props.storageDirectory);	
+		return dataDir.listFiles(filter).length;
+	}
+
+
 	public void setProps(PropertiesHolder props) {
 		this.props = props;
 	}

src/main/java/com/rpsgroup/HarvesterMain.java

@@ -6,6 +6,7 @@
 import java.net.MalformedURLException;
 import java.net.URL;
 import java.nio.file.Files;
+import java.nio.file.Path;
 import java.text.SimpleDateFormat;
 import java.util.Date;
 
@@ -26,6 +27,7 @@ public class HarvesterMain {
 	public static void main(String[] args) throws MalformedURLException, Exception {
 
 		HarvesterMain runner = new HarvesterMain();	
+		File dodsFile = null;
 
 		try {
 			runner.converter.setProps(props);
@@ -44,20 +46,25 @@ public static void main(String[] args) throws MalformedURLException, Exception {
 			Boolean firstRun = mostRecentLocal.compareTo(new Date(0L)) == 0;
 
 			if (remoteModTime.after(mostRecentLocal) || firstRun) {
+				runner.log.info("Downloading latest data...");
 
-				File dodsFile = runner.http.downloadDods();
+				dodsFile = runner.http.downloadDods();
 				runner.log.info("Downloaded .dods file: " + dodsFile.toString());
 
 				File netcdf = runner.file.getNC4Outfile();
 				runner.converter.convert(dodsFile, netcdf);
 
-				runner.log.info("Converted .dods to netCDF4: " + netcdf.toString());
-				Files.delete(dodsFile.toPath());
-
-				runner.log.info("Deleted .dods file");
+				runner.log.info("Converted .dods to netCDF4: " + netcdf.toString());				
+				while (runner.file.getFileCount() > props.storageMaxCount) {	
+
+					Path oldest = runner.file.getOldestFile().toPath();
+					runner.log.info("Exceeded storage max count, deleting oldest: " + oldest);
+
+					Files.delete(oldest);
+				}
 
 			} else {
-				runner.log.info("Nothing to do here, we have latest data. Goodbye.");
+				runner.log.info("Nothing to do here, we have latest data.");
 			}
 
 		} catch (Exception ex) {
@@ -67,6 +74,14 @@ public static void main(String[] args) throws MalformedURLException, Exception {
 
 			ex.printStackTrace(pw);
 			runner.log.error(sw.toString());
+
+		} finally {
+
+			if (dodsFile != null) {
+				Files.delete(dodsFile.toPath());
+				runner.log.info("Deleted .dods file");
+			}
+			runner.log.info("Done.");
 		}
 
 		// 4. Run compliance checks.

src/main/java/com/rpsgroup/PropertiesHolder.java

@@ -15,6 +15,7 @@ public class PropertiesHolder {
 	public String downloadDirectory;
 	public String storageDirectory;
 	public String jnaLibDirectory;
+	public int storageMaxCount;
 
 	public PropertiesHolder() {
 
@@ -39,6 +40,7 @@ public PropertiesHolder() {
 		downloadDirectory = getProp("directory.download");
 		storageDirectory = getProp("directory.storage");
 		jnaLibDirectory = getProp("directory.jna.lib");
+		storageMaxCount = Integer.parseInt(getProp("directory.storage.maxcount"));
 
 	}
 

src/main/resources/application.properties

@@ -9,7 +9,12 @@ nc.filename=fvcom_lis_forecast.nc
 directory.download=/tmp
 
 ## Final resting place for harvested data
-directory.storage=/tmp
+#directory.storage=/tmp
+directory.storage=/data/maracoos/uconn/fvcom-lis
+
+## Number of past forecasts to keep
+directory.storage.maxcount=90
 
 ## Required path the netCDF-C shared libraries
-directory.jna.lib=/usr/local/Cellar/netcdf/4.8.0_2/lib
+#directory.jna.lib=/usr/local/Cellar/netcdf/4.8.0_2/lib
+directory.jna.lib=/opt/netcdf-4.3.0/lib

src/main/resources/logback.xml

@@ -1,32 +1,22 @@
 <?xml version="1.0" encoding="UTF-8"?>
 <configuration>
-
 	<appender name="FILE" class="ch.qos.logback.core.FileAppender">
-
 		<file>/var/log/harvest/harvest.log</file>
-
 		<encoder>
 			<pattern>%date %level [%thread] %logger{10} [%file:%line] %msg%n
 			</pattern>
 		</encoder>
-
 	</appender>
-
 	<appender name="STDOUT"
 		class="ch.qos.logback.core.ConsoleAppender">
-
 		<encoder>
 			<pattern>%date %level [%thread] %logger{10} [%file:%line] %msg%n
 			</pattern>
 		</encoder>
-
 	</appender>
 
 	<root level="trace">
-
 		<appender-ref ref="FILE" />
 		<appender-ref ref="STDOUT" />
-
 	</root>
-
 </configuration>

unpack-deploy.sh

@@ -0,0 +1,8 @@
+#!/bin/bash
+rm -rf $(ls | grep -vE '(\.sh|tar\.gz)')
+tar -xf ./*.tar.gz
+rm -f ./*.tar.gz
+unzip dods-harvester*.jar
+mkdir lib/
+mv *.jar lib/
+