Merge branch 'dev'

README.md

@@ -36,7 +36,7 @@ Heres a list of flags that `auto.py` can accept:
 | --- | --- |
 | `--dayonly` | Do not take nighttime screenshots (For now, this setting needs to be the same across one timeline). |
 | `--nightonly` | Do not take daytime screenshots. |
-| `--HD` | Take screenshots of resolution 64 x 64 pixels per in-game tile instead of 32 x 32 to match the resolution of the newer HD textures. |
+| `--hd` | Take screenshots of resolution 64 x 64 pixels per in-game tile instead of 32 x 32 to match the resolution of the newer HD textures. |
 | `--no-altmode` | Hides entity info (alt mode) |
 | `--build-range=5.2` | The maximum range from buildings around which pictures are saved (in chunks, 32 by 32 in-game tiles). |
 | `--connect-range=1.2` | The maximum range from connection buildings (rails, electric poles) around which pictures are saved. |
@@ -51,6 +51,8 @@ Heres a list of flags that `auto.py` can accept:
 | `--cropthreads=N` | Sets the number of threads used for the crop step. |
 | `--refthreads=N` | Sets the number of threads used for the crossreferencing step. |
 | `--zoomthreads=N` | Sets the number of threads used for the zoom step. |
+| `--screenshotthreads=N` | Set the number of screenshotting threads factorio uses. |
+| `--screenshotqueuesize=N` | Set the size of the factorio screenshotting queue. Too high values may cause high RAM usage and I haven't found any benefits to high values so I recommend not changing this. |
 | `--delete` | Deletes the output folder specified before running the script. |
 | `--dry` | Skips starting factorio, making screenshots and doing the main steps, only execute setting up and finishing of script. |
 

auto.py

@@ -6,13 +6,15 @@
 import re
 import random
 import math
+import configparser
 from subprocess import call
 import datetime
 import urllib.request, urllib.error, urllib.parse
 from shutil import copy, rmtree, get_terminal_size as tsize
 from zipfile import ZipFile
 import tempfile
 from PIL import Image, ImageChops
+import multiprocessing as mp
 
 from crop import crop
 from ref import ref
@@ -23,18 +25,34 @@
 def auto(*args):
 
 
+
+
+
 	def printErase(arg):
 		try:
 			tsiz = tsize()[0]
 			print("\r{}{}\n".format(arg, " " * (tsiz*math.ceil(len(arg)/tsiz)-len(arg) - 1)), end="", flush=True)
 		except e:
-			raise e
-			try:
-				print("PRINTERROR %s" % arg)
-			except:
-				pass
+			#raise
+			pass
+
+
+
+	def kill(pid):
+		if psutil.pid_exists(pid):
+
+			if os.name == 'nt':
+				cmd = ("taskkill", "/pid", str(pid))
+			else:
+				cmd = ("kill", str(pid))
+			subprocess.check_call(cmd, stdout=subprocess.DEVNULL, shell=True)
+
+			while psutil.pid_exists(pid):
+				time.sleep(0.1)
 
+			printErase("killed factorio")
 
+		time.sleep(0.1)
 
 
 	def parseArg(arg):
@@ -61,11 +79,11 @@ def parseArg(arg):
 		"D:/Games/Factorio/bin/x64/factorio.exe",
 		"E:/Games/Factorio/bin/x64/factorio.exe",
 		"F:/Games/Factorio/bin/x64/factorio.exe",
+		"../../bin/x64/factorio",
 		"C:/Program Files (x86)/Steam/steamapps/common/Factorio/bin/x64/factorio.exe",
 		"D:/Program Files (x86)/Steam/steamapps/common/Factorio/bin/x64/factorio.exe",
 		"E:/Program Files (x86)/Steam/steamapps/common/Factorio/bin/x64/factorio.exe",
-		"F:/Program Files (x86)/Steam/steamapps/common/Factorio/bin/x64/factorio.exe",
-		"../../bin/x64/factorio"
+		"F:/Program Files (x86)/Steam/steamapps/common/Factorio/bin/x64/factorio.exe"
 	]
 	try:
 		factorioPath = next(x for x in map(os.path.abspath, [kwargs["factorio"]] if "factorio" in kwargs else possiblePaths) if os.path.isfile(x))
@@ -158,9 +176,10 @@ def parseArg(arg):
 
 	def linkDir(src, dest):
 		if os.name == 'nt':
-			subprocess.call(("MKLINK", "/J", os.path.abspath(src), os.path.abspath(dest)), shell=True)
+			cmd = ("MKLINK", "/J", os.path.abspath(src), os.path.abspath(dest))
 		else:
-			subprocess.call(("ln", "-s", os.path.abspath(src), os.path.abspath(dest)), shell=True)
+			cmd = ("ln", "-s", os.path.abspath(src), os.path.abspath(dest))
+		subprocess.check_call(cmd, stdout=subprocess.DEVNULL, shell=True)
 
 	print("enabling FactorioMaps mod")
 	modListPath = os.path.join(kwargs["modpath"], "mod-list.json") if "modpath" in kwargs else "../mod-list.json"
@@ -211,14 +230,23 @@ def printGameLog(pipe):
 					printErase("[GAME] %s" % line)
 
 
+
+
+	if "delete" in kwargs:
+		try:
+			rmtree(workfolder)
+		except (FileNotFoundError, NotADirectoryError):
+			pass
+
+
+
 	logIn, logOut = os.pipe()
 	logthread = threading.Thread(target=printGameLog, args=[logIn])
 	logthread.daemon = True
 	logthread.start()
 
 
 
-
 	datapath = os.path.join(workfolder, "latest.txt")
 	allTmpDirs = []
 
@@ -233,14 +261,6 @@ def printGameLog(pipe):
 				os.remove(datapath)
 
 
-
-			if "delete" in kwargs:
-				try:
-					rmtree(workfolder)
-				except (FileNotFoundError, NotADirectoryError):
-					pass
-
-
 
 			printErase("building autorun.lua")
 			if (os.path.isfile(os.path.join(workfolder, "mapInfo.json"))):
@@ -274,34 +294,53 @@ def printGameLog(pipe):
 
 
 			printErase("starting factorio")
-			tmpdir = os.path.join(tempfile.gettempdir(), "FactorioMaps-%s" % random.randint(1, 999999999))
-			allTmpDirs.append(tmpdir)
+			tmpDir = os.path.join(tempfile.gettempdir(), "FactorioMaps-%s" % random.randint(1, 999999999))
+			allTmpDirs.append(tmpDir)
 			try:
-				rmtree(tmpdir)
+				rmtree(tmpDir)
 			except (FileNotFoundError, NotADirectoryError):
 				pass
-			os.makedirs(os.path.join(tmpdir, "config"))
-			configPath = os.path.join(tmpdir, "config/config.ini")
-			configInserted = False
-			with open(configPath, 'w+') as outf, open("../../config/config.ini", "r") as inf:
-				for line in inf:
-					if re.match(r'^ *write-data *=.*', line, re.IGNORECASE) is None:
-						outf.write(line)
-					if not configInserted and re.match(r'^ *\[path\].*', line, re.IGNORECASE) is not None:
-						outf.write("write-data=%s\n" % tmpdir)
-						configInserted = True
-				if not configInserted:
-					outf.write("[path]\n")
-					outf.write("write-data=%s\n" % tmpdir)
-
-			linkDir(os.path.join(tmpdir, "script-output"), "../../script-output")
-			copy("../../player-data.json", os.path.join(tmpdir, "player-data.json"))
+			os.makedirs(os.path.join(tmpDir, "config"))
 
+			configPath = os.path.join(tmpDir, "config/config.ini")
+			config = configparser.ConfigParser()
+			config.read("../../config/config.ini")
+
+			config["interface"]["show-tips-and-tricks"] = "false"
+
+			config["path"]["write-data"] = tmpDir
+			config["graphics"]["screenshots-threads-count"] = str(int(kwargs["screenshotthreads"]) if "screenshotthreads" in kwargs else (int(kwargs["maxthreads"]) if "maxthreads" in kwargs else mp.cpu_count()))
+			config["graphics"]["max-threads"] = config["graphics"]["screenshots-threads-count"]
+			config["graphics"]["screenshots-queue-size"] = str(int(kwargs["screenshotqueuesize"]) if "screenshotqueuesize" in kwargs else mp.cpu_count()*2)
 
+			with open(configPath, 'w+') as outf:
+				outf.writelines(("; version=3\n", ))
+				config.write(outf, space_around_delimiters=False)
+
+
+			linkDir(os.path.join(tmpDir, "script-output"), "../../script-output")
+			copy("../../player-data.json", os.path.join(tmpDir, "player-data.json"))
+
+			pid = None
+			pidBlacklist = [p.info["pid"] for p in psutil.process_iter(attrs=['pid', 'name']) if p.info['name'] == "factorio.exe"]
+
 			p = subprocess.Popen((factorioPath, '--load-game', os.path.abspath(os.path.join("../../saves", savename+".zip")), '--disable-audio', '--config', configPath, "--mod-directory", os.path.abspath(kwargs["modpath"] if "modpath" in kwargs else "../../mods")), stdout=logOut)
 			time.sleep(1)
 			if p.poll() is not None:
-				print("WARNING: running in limited support mode trough steam. Consider using standalone factorio instead.\n\tPlease confirm the steam 'start game with arguments' popup.")
+				print("WARNING: running in limited support mode trough steam. Consider using standalone factorio instead.\n\t Please open steam and confirm the steam 'start game with arguments' popup.")
+				attrs = ('pid', 'name', 'create_time')
+				oldest = None
+				while pid is None:
+					for proc in psutil.process_iter(attrs=attrs):
+						pinfo = proc.as_dict(attrs=attrs)
+						if pinfo["name"] == "factorio.exe" and pinfo["pid"] not in pidBlacklist and (pid is None or pinfo["create_time"] < oldest):
+							oldest = pinfo["create_time"]
+							pid = pinfo["pid"]
+					if pid is None:
+						time.sleep(1)
+			else:
+				pid = p.pid
+
 
 			if not os.path.exists(datapath):
 				while not os.path.exists(datapath):
@@ -319,23 +358,16 @@ def printGameLog(pipe):
 
 
 			isKilled = [False]
-			def waitKill(isKilled):
+			def waitKill(isKilled, pid):
 				while not isKilled[0]:
 					if os.path.isfile(waitfilename):
 						isKilled[0] = True
-						if p.poll() is None:
-							p.kill()
-						else:
-							if os.name == 'nt':
-								os.system("taskkill /im factorio.exe")
-							else:
-								os.system("killall factorio")
-						printErase("killed factorio")
+						kill(pid)
 						break
 					else:
 						time.sleep(0.4)
 
-			killthread = threading.Thread(target=waitKill, args=(isKilled,))
+			killthread = threading.Thread(target=waitKill, args=(isKilled, pid))
 			killthread.daemon = True
 			killthread.start()
 
@@ -374,14 +406,7 @@ def refZoom():
 				else:
 					if not isKilled[0]:
 						isKilled[0] = True
-						if p.poll() is None:
-							p.kill()
-						else:
-							if os.name == 'nt':
-								os.system("taskkill /im factorio.exe")
-							else:
-								os.system("killall factorio")
-						printErase("killed factorio")
+						kill(pid)
 
 					if savename == savenames[-1]:
 						refZoom()
@@ -417,7 +442,8 @@ def refZoom():
 			for mapStuff in data["maps"]:
 				for surfaceName, surfaceStuff in mapStuff["surfaces"].items():
 					for tag in surfaceStuff["tags"]:
-						tags[tag["iconType"] + tag["iconName"][0].upper() + tag["iconName"][1:]] = tag
+						if "iconType" in tag:
+							tags[tag["iconType"] + tag["iconName"][0].upper() + tag["iconName"][1:]] = tag
 
 		rmtree(os.path.join(workfolder, "Images", "labels"), ignore_errors=True)
 
@@ -484,7 +510,7 @@ def refZoom():
 
 
 
-
+		#TODO: download leaflet shit
 
 		print("generating mapInfo.js")
 		with open(os.path.join(workfolder, "mapInfo.js"), 'w') as outf, open(os.path.join(workfolder, "mapInfo.json"), "r") as inf:
@@ -499,17 +525,14 @@ def refZoom():
 
 
 	except KeyboardInterrupt:
-		if p.poll() is None:
-			p.kill()
-		else:
-			if os.name == 'nt':
-				os.system("taskkill /im factorio.exe")
-			else:
-				os.system("killall factorio")
-		print("killed factorio")
+		print("keyboardinterrupt")
+		kill(pid)
 		raise
 
 	finally:
+
+		kill(pid)
+
 		print("disabling FactorioMaps mod")
 		changeModlist(False)
 
@@ -519,7 +542,7 @@ def refZoom():
 		open("autorun.lua", 'w').close()
 		for tmpDir in allTmpDirs:
 			try:
-				os.unlink(os.path.join(tmpdir, "script-output"))
+				os.unlink(os.path.join(tmpDir, "script-output"))
 				rmtree(tmpDir)
 			except (FileNotFoundError, NotADirectoryError):
 				pass

control.lua

@@ -22,6 +22,9 @@ script.on_event(defines.events.on_tick, function(event)
 		event.player_index = game.connected_players[1].index
 
 		if nil == fm.tmp then
+
+			log("Start world capture")
+
 			-- freeze all entities. Eventually, stuff will run out of power, but for just 2 ticks, it should be fine.
 			for key, entity in pairs(game.connected_players[1].surface.find_entities_filtered({invert=true, name="hidden-electric-energy-interface"})) do
 				entity.active = false

generateMap.lua

@@ -57,6 +57,8 @@ function fm.generateMap(data)
 	game.remove_path(subPath)
 	subPath = subPath .. "/"
 
+	log("Starting surface prescan to target directory: " .. subPath)
+
 
 
 	-- Number of pixels in an image     -- CHANGE THIS AND REF.PY WILL NEED TO BE CHANGED
@@ -338,14 +340,23 @@ function fm.generateMap(data)
 			tags = {}
 		}
 		for i, tag in pairs(force.find_chart_tags(surface)) do
-			fm.autorun.mapInfo.maps[mapIndex].surfaces[surface.name].tags[i] = {
-				iconType 	= tag.icon.type,
-				iconName 	= tag.icon.name or tag.icon.type,
-				iconPath    = "Images/labels/" .. tag.icon.type .. "/" .. tag.icon.name .. ".png",
-				position 	= tag.position,
-				text 		= tag.text,
-				last_user	= tag.last_user and tag.last_user.name
-			}
+			if tag.icon == nil then
+				fm.autorun.mapInfo.maps[mapIndex].surfaces[surface.name].tags[i] = {
+					position 	= tag.position,
+					text 		= tag.text,
+					last_user	= tag.last_user and tag.last_user.name
+				}
+			else
+				name = tag.icon["name"] or tag.icon.type
+				fm.autorun.mapInfo.maps[mapIndex].surfaces[surface.name].tags[i] = {
+					iconType 	= tag.icon.type,
+					iconName 	= name,
+					iconPath    = "Images/labels/" .. tag.icon.type .. "/" .. name .. ".png",
+					position 	= tag.position,
+					text 		= tag.text,
+					last_user	= tag.last_user and tag.last_user.name
+				}
+			end
 		end
 
 		if fm.autorun.chunkCache[fm.autorun.tick] == nil then
@@ -381,6 +392,10 @@ function fm.generateMap(data)
 
 	local extension = "bmp"
 
+
+
+	log("Starting surface capture to target directory: " .. subPath)
+
 
 	game.write_file(basePath .. "/mapInfo.json", json(fm.autorun.mapInfo), false, data.player_index)