Add FTPS support and other FTP improvements

FtpWriteFile.py

@@ -4,7 +4,9 @@
 import os
 import sys
 import webbrowser
+import ftplib
 import ftputil
+import ftputil.session
 import paramiko
 from urllib.parse import quote
 import datetime
@@ -21,10 +23,10 @@ def lineno():
 	return inspect.currentframe().f_back.f_lineno
 
 class CallCloseOnExit:
-	def __enter__(self, obj):
+	def __init__(self, obj):
 		self.obj = obj
-		return obj
-
+	def __enter__(self):
+		return self.obj
 	def __exit__(self, exc_type, exc_val, exc_tb):
 		self.obj.close()
 
@@ -55,16 +57,43 @@ def sftp_mkdir_p( sftp, remote_directory ):
 	# Create new dirs starting from the last one that existed.
 	for i in range( i_dir_last, len(dirs_exist) ):
 		sftp.mkdir( '/'.join(dirs_exist[:i+1]) )
-
-def FtpWriteFile( host, user='anonymous', passwd='anonymous@', timeout=30, serverPath='.', fname='', useSftp=False, sftpPort=22, callback=None ):
+
+
+class FtpWithPort(ftplib.FTP):
+	def __init__(self, host, user, passwd, port, timeout):
+		#Act like ftplib.FTP's constructor but connect to another port.
+		ftplib.FTP.__init__(self)
+		#self.set_debuglevel(2)
+		self.connect(host, port, timeout)
+		self.login(user, passwd)
+
+class FtpsWithPort(ftplib.FTP_TLS):
+	def __init__(self, host, user, passwd, port, timeout):
+		ftplib.FTP_TLS.__init__(self)
+		#self.set_debuglevel(2)
+		self.connect(host, port, timeout)
+		self.auth()
+		self.login(user, passwd)
+		#Switch to secure data connection.
+		self.prot_p()
+
+	def ntransfercmd(self, cmd, rest=None):
+		conn, size = ftplib.FTP.ntransfercmd(self, cmd, rest)
+		if self._prot_p:
+			conn = self.context.wrap_socket(conn,
+											server_hostname=self.host,
+											session=self.sock.session)  #reuse ssl session
+		return conn, size
+
+def FtpWriteFile( host, port, user='anonymous', passwd='anonymous@', timeout=30, serverPath='.', fname='', protocol='FTP', callback=None ):
 
 	if isinstance(fname, str):
 		fname = [fname]
 
 	# Normalize serverPath.
 	serverPath = serverPath.strip().replace('\\', '/').rstrip('/')
 
-	if not useSftp:
+	if protocol != 'SFTP':
 		# Stops ftputils from going into an infinite loop by removing leading slashes..
 		serverPath = serverPath.lstrip('/').lstrip('\\')
 
@@ -85,29 +114,47 @@ def FtpWriteFile( host, user='anonymous', passwd='anonymous@', timeout=30, serve
 		return
 	'''
 
-	if useSftp:
+	if protocol == 'SFTP':
 		with CallCloseOnExit(paramiko.SSHClient()) as ssh:
 			ssh.set_missing_host_key_policy( paramiko.AutoAddPolicy() )
 			ssh.load_system_host_keys()                  
-			ssh.connect( host, sftpPort, username, passwd )
+			ssh.connect( host, port, user, passwd )
 
 			with CallCloseOnExit(ssh.open_sftp()) as sftp:
 				sftp_mkdir_p( sftp, serverPath )
 				for i, f in enumerate(fname):
 					sftp.put(
-						filePath,
+						f,
 						serverPath + '/' + os.path.basename(f),
 						SftpCallback( callback, f, i ) if callback else None
 					)
-	else:
-		with ftputil.FTPHost( host, user, passwd ) as ftp_host:
+	elif protocol == 'FTPS':
+		with ftputil.FTPHost(host, user, passwd, port, timeout, session_factory=FtpsWithPort) as ftp_host:
+			ftp_host.makedirs( serverPath, exist_ok=True )
+			for i, f in enumerate(fname):
+				try:
+					ftp_host.upload_if_newer(
+						f,
+						serverPath + '/' + os.path.basename(f),
+						(lambda byteStr, fname=f, i=i: callback(byteStr, fname, i)) if callback else None
+					)
+				except ftplib.all_errors as e:
+					if 'EOF occurred in violation of protocol' in str(e):
+						Utils.writeLog( 'FtpWriteFile ignored \"' + str(e) + '\" as this can be non-fatal.  Check if the file exists on the server.')
+					else:
+						raise e
+			ftp_host.close()
+
+	else:  #default to unencrypted FTP
+		with ftputil.FTPHost(host, user, passwd, port, timeout, session_factory=FtpWithPort) as ftp_host:
 			ftp_host.makedirs( serverPath, exist_ok=True )
 			for i, f in enumerate(fname):
 				ftp_host.upload_if_newer(
 					f,
 					serverPath + '/' + os.path.basename(f),
 					(lambda byteStr, fname=f, i=i: callback(byteStr, fname, i)) if callback else None
 				)
+			ftp_host.close()
 
 def FtpIsConfigured():
 	with Model.LockRace() as race:
@@ -126,10 +173,11 @@ def FtpUploadFile( fname=None, callback=None ):
 
 	params = {
 		'host': 		getattr(race, 'ftpHost', '').strip().strip('\t'),	# Fix cut and paste problems.
+		'port':			getattr(race, 'ftpPort', 21),
 		'user':			getattr(race, 'ftpUser', ''),
 		'passwd':		getattr(race, 'ftpPassword', ''),
 		'serverPath':	getattr(race, 'ftpPath', ''),
-		'useSftp':		getattr(race, 'useSftp', False),
+		'protocol':		getattr(race, 'ftpProtocol', 'FTP'),
 		'fname':		fname or [],
 		'callback':		callback,
 	}
@@ -350,8 +398,8 @@ def getTitleTextSize( font ):
 
 #------------------------------------------------------------------------------------------------
 
-ftpFields = 	['ftpHost',	'ftpPath',	'ftpPhotoPath',	'ftpUser',		'ftpPassword',	'useSftp',	'ftpUploadDuringRace',	'urlPath', 'ftpUploadPhotos']
-ftpDefaults =	['',		'',			'',				'anonymous',	'anonymous@',	False,		False,					'http://',	False]
+ftpFields = 	['ftpHost',	'ftpPort', 'ftpPath',	'ftpPhotoPath',	'ftpUser',		'ftpPassword',	'ftpUploadDuringRace',	'urlPath', 'ftpUploadPhotos']
+ftpDefaults =	['',	21,	'',			'',				'anonymous',	'anonymous@',	False,						'http://',	False]
 
 def GetFtpPublish( isDialog=True ):
 	ParentClass = wx.Dialog if isDialog else wx.Panel
@@ -364,11 +412,17 @@ def __init__( self, parent, id=wx.ID_ANY, uploadNowButton=True ):
 			else:
 				super().__init__( parent, id )
 
+			self.protocol = 'FTP'
+
 			fgs = wx.FlexGridSizer(vgap=4, hgap=4, rows=0, cols=2)
 			fgs.AddGrowableCol( 1, 1 )
 
-			self.useSftp = wx.CheckBox( self, label=_("Use SFTP Protocol (on port 22)") )
+			self.useFtp = wx.RadioButton( self, label=_("FTP (unencrypted)"), style = wx.RB_GROUP )
+			self.useFtps = wx.RadioButton( self, label=_("FTPS (FTP with TLS)") )
+			self.useSftp = wx.RadioButton( self, label=_("SFTP (SSH file transfer)") )
+			self.Bind( wx.EVT_RADIOBUTTON,self.onSelectProtocol ) 
 			self.ftpHost = wx.TextCtrl( self, size=(256,-1), style=wx.TE_PROCESS_ENTER, value='' )
+			self.ftpPort = wx.lib.intctrl.IntCtrl( self, size=(256,-1), style=wx.TE_PROCESS_ENTER )
 			self.ftpPath = wx.TextCtrl( self, size=(256,-1), style=wx.TE_PROCESS_ENTER, value='' )
 			self.ftpUploadPhotos = wx.CheckBox( self, label=_("Upload Photos to Path") )
 			self.ftpUploadPhotos.Bind( wx.EVT_CHECKBOX, self.ftpUploadPhotosChanged )
@@ -393,12 +447,20 @@ def __init__( self, parent, id=wx.ID_ANY, uploadNowButton=True ):
 				self.cancelBtn = wx.Button( self, wx.ID_CANCEL )
 				self.Bind( wx.EVT_BUTTON, self.onCancel, self.cancelBtn )
 
+			fgs.Add( wx.StaticText( self, label = _("Protocol")), flag=wx.ALIGN_RIGHT|wx.ALIGN_CENTRE_VERTICAL )
+			fgs.Add( self.useFtp, 1, flag=wx.TOP|wx.ALIGN_LEFT)
+			fgs.AddSpacer( 16 )
+			fgs.Add( self.useFtps, 1, flag=wx.TOP|wx.ALIGN_LEFT)
 			fgs.AddSpacer( 16 )
-			fgs.Add( self.useSftp )
+			fgs.Add( self.useSftp, 1, flag=wx.TOP|wx.ALIGN_LEFT)
+
 
 			fgs.Add( wx.StaticText( self, label = _("Host Name")), flag=wx.ALIGN_RIGHT|wx.ALIGN_CENTRE_VERTICAL )
 			fgs.Add( self.ftpHost, 1, flag=wx.TOP|wx.ALIGN_LEFT|wx.EXPAND )
 
+			fgs.Add( wx.StaticText( self, label = _("Port")), flag=wx.ALIGN_RIGHT|wx.ALIGN_CENTRE_VERTICAL )
+			fgs.Add( self.ftpPort, 1, flag=wx.TOP|wx.ALIGN_LEFT|wx.EXPAND )
+
 			fgs.Add( wx.StaticText( self, label = _("Upload files to Path")),  flag=wx.ALIGN_RIGHT|wx.ALIGN_CENTRE_VERTICAL )
 			fgs.Add( self.ftpPath, 1, flag=wx.EXPAND )
 
@@ -459,6 +521,17 @@ def __init__( self, parent, id=wx.ID_ANY, uploadNowButton=True ):
 					fgs.AddSpacer( 4 )
 				self.SetSizerAndFit( fgs )
 				fgs.Fit( self )
+
+		def onSelectProtocol( self, event ):
+			if self.useSftp.GetValue():
+				self.protocol = 'SFTP'
+				self.ftpPort.SetValue(22)
+			elif self.useFtps.GetValue():
+				self.protocol = 'FTPS'
+				self.ftpPort.SetValue(21)
+			else:
+				self.protocol = 'FTP'
+				self.ftpPort.SetValue(21)
 
 		def onFtpTest( self, event ):
 			self.commit()
@@ -529,8 +602,17 @@ def refresh( self ):
 				for f, v in zip(ftpFields, ftpDefaults):
 					getattr(self, f).SetValue( v )
 			else:
+				self.protocol = getattr(race, 'ftpProtocol', '')
+				if self.protocol == 'SFTP':
+					self.useSftp.SetValue(True)
+				elif self.protocol == 'FTPS':
+					self.useFtps.SetValue(True)
+				else:
+					self.useFtp.SetValue(True)
 				for f, v in zip(ftpFields, ftpDefaults):
 					getattr(self, f).SetValue( getattr(race, f, v) )
+
+
 			self.urlPathChanged()
 			self.ftpUploadPhotosChanged()
 
@@ -540,6 +622,7 @@ def commit( self ):
 			if race:
 				for f in ftpFields:
 					setattr( race, f, getattr(self, f).GetValue() )
+				setattr( race, 'ftpProtocol', self.protocol)
 				race.urlFull = self.urlFull.GetLabel()
 			race.setChanged()
 

Properties.py

@@ -818,28 +818,28 @@ def commit( self ):
 
 #------------------------------------------------------------------------------------------------
 class BatchPublishProperties( wx.Panel ):
-	def __init__( self, parent, id=wx.ID_ANY, testCallback=None, ftpCallback=None ):
+	def __init__( self, parent, id=wx.ID_ANY, publishCallback=None, ftpCallback=None ):
 		super().__init__( parent, id )
 
-		self.testCallback = testCallback
+		self.publishCallback = publishCallback
 		self.ftpCallback = ftpCallback
 
 		if ftpCallback:
-			ftpBtn = wx.ToggleButton( self, label=_('Configure Ftp') )
+			ftpBtn = wx.ToggleButton( self, label=_('Configure FTP') )
 			ftpBtn.Bind( wx.EVT_TOGGLEBUTTON, ftpCallback )
 		else:
 			ftpBtn = None
 
 		explain = [
-			wx.StaticText(self,label=_('Choose File Formats to Publish.  Select Ftp option to upload files to Ftp server.')),
+			wx.StaticText(self,label=_('Choose File Formats to Publish.  Select FTP option to upload files to (S)FTP server.')),
 		]
 		font = explain[0].GetFont()
 		fontUnderline = wx.FFont( font.GetPointSize(), font.GetFamily(), flags=wx.FONTFLAG_BOLD )
 
 		fgs = wx.FlexGridSizer( cols=4, rows=0, hgap=0, vgap=1 )
 		self.widget = []
 
-		headers = [_('Format'), _('Ftp'), _('Note'), '']
+		headers = [_('Format'), _('FTP'), _('Note'), '']
 		for h in headers:
 			st = wx.StaticText(self, label=h)
 			st.SetFont( fontUnderline )
@@ -862,10 +862,10 @@ def __init__( self, parent, id=wx.ID_ANY, testCallback=None, ftpCallback=None ):
 			else:
 				fgs.AddSpacer( 0 )
 
-			testBtn = wx.Button( self, label=_('Publish') )
-			testBtn.Bind( wx.EVT_BUTTON, lambda event, iAttr=i: self.onTest(iAttr) )
-			fgs.Add( testBtn, flag=wx.LEFT|wx.ALIGN_CENTRE_VERTICAL, border=8 )
-			self.widget.append( (attrCB, ftpCB, testBtn) )
+			publishBtn = wx.Button( self, label=_('Publish') )
+			publishBtn.Bind( wx.EVT_BUTTON, lambda event, iAttr=i: self.onPublish(iAttr) )
+			fgs.Add( publishBtn, flag=wx.LEFT|wx.ALIGN_CENTRE_VERTICAL, border=8 )
+			self.widget.append( (attrCB, ftpCB, publishBtn) )
 
 		self.bikeRegChoice = wx.RadioBox(
 			self,
@@ -902,11 +902,11 @@ def __init__( self, parent, id=wx.ID_ANY, testCallback=None, ftpCallback=None ):
 
 		self.SetSizer( vs )
 
-	def onTest( self, iAttr ):
-		if self.testCallback:
-			self.testCallback()
+	def onPublish( self, iAttr ):
+		if self.publishCallback:
+			self.publishCallback()
 
-		attrCB, ftpCB, testBtn = self.widget[iAttr]
+		attrCB, ftpCB, publishBtn = self.widget[iAttr]
 		doFtp = ftpCB and ftpCB.GetValue()
 		doBatchPublish( iAttr, silent=False )
 
@@ -917,7 +917,7 @@ def onTest( self, iAttr ):
 		if attr.filecode:
 			fname = mainWin.getFormatFilename(attr.filecode)
 			if doFtp and race.urlFull and race.urlFull != 'http://':
-				webbrowser.open( os.path.basename(race.urlFull) + '/' + os.path.basename(fname), new = 0, autoraise = True )
+				webbrowser.open( race.urlFull, new = 0, autoraise = True )
 			else:
 				Utils.LaunchApplication( fname )
 		else:
@@ -927,40 +927,40 @@ def onTest( self, iAttr ):
 				return
 
 	def onSelect( self, iAttr ):
-		attrCB, ftpCB, testBtn = self.widget[iAttr]
+		attrCB, ftpCB, publishBtn = self.widget[iAttr]
 		v = attrCB.GetValue()
 		if ftpCB:
 			ftpCB.Enable( v )
 			if not v:
 				ftpCB.SetValue( False )
-		testBtn.Enable( v )
+		publishBtn.Enable( v )
 
 	def refresh( self ):
 		race = Model.race
 		for i, attr in enumerate(batchPublishAttr):
 			raceAttr = batchPublishRaceAttr[i]
-			attrCB, ftpCB, testBtn = self.widget[i]
+			attrCB, ftpCB, publishBtn = self.widget[i]
 			v = getattr( race, raceAttr, 0 )
 			if v & 1:
 				attrCB.SetValue( True )
 				if ftpCB:
 					ftpCB.Enable( True )
 					ftpCB.SetValue( v & 2 != 0 )
-				testBtn.Enable( True )
+				publishBtn.Enable( True )
 			else:
 				attrCB.SetValue( False )
 				if ftpCB:
 					ftpCB.SetValue( False )
 					ftpCB.Enable( False )
-				testBtn.Enable( False )
+				publishBtn.Enable( False )
 		self.bikeRegChoice.SetSelection( getattr(race, 'publishFormatBikeReg', 0) )
 		self.postPublishCmd.SetValue( race.postPublishCmd )
 
 	def commit( self ):
 		race = Model.race
 		for i, attr in enumerate(batchPublishAttr):
 			raceAttr = batchPublishRaceAttr[i]
-			attrCB, ftpCB, testBtn = self.widget[i]
+			attrCB, ftpCB, publishBtn = self.widget[i]
 			setattr( race, raceAttr, 0 if not attrCB.GetValue() else (1 + (2 if ftpCB and ftpCB.GetValue() else 0)) )
 		race.publishFormatBikeReg = self.bikeRegChoice.GetSelection()
 		race.postPublishCmd = self.postPublishCmd.GetValue().strip()
@@ -1076,7 +1076,7 @@ def __init__( self, parent, id=wx.ID_ANY ):
 		super().__init__( parent, id, _("Batch Publish Results"),
 					style=wx.DEFAULT_DIALOG_STYLE|wx.TAB_TRAVERSAL )
 
-		self.batchPublishProperties = BatchPublishProperties(self, testCallback=self.commit, ftpCallback=self.onToggleFtp)
+		self.batchPublishProperties = BatchPublishProperties(self, publishCallback=self.commit, ftpCallback=self.onToggleFtp)
 		self.batchPublishProperties.refresh()
 
 		self.ftp = FtpProperties( self, uploadNowButton=False )
@@ -1273,7 +1273,7 @@ def __init__( self, parent, id=wx.ID_ANY, addEditButton=True ):
 			('raceOptionsProperties',	RaceOptionsProperties,		_('Race Options') ),
 			('rfidProperties',			RfidProperties,				_('RFID') ),
 			('webProperties',			WebProperties,				_('Web') ),
-			('ftpProperties',			FtpProperties,				_('FTP') ),
+			('ftpProperties',			FtpProperties,				_('(S)FTP') ),
 			('batchPublishProperties',	BatchPublishProperties,		_('Batch Publish') ),
 			('gpxProperties',			GPXProperties,				_('GPX') ),
 			('notesProperties',			NotesProperties,			_('Notes') ),