From c019c52f388dffd609ec3d8baefcdeeb9e462ec1 Mon Sep 17 00:00:00 2001 From: Anairkoen Schno Date: Sat, 4 Aug 2018 23:56:54 -0500 Subject: [PATCH] The updater works, dont worry. Really this time. --- IPA.Tests/updater_test.json | 28 +-- IPA/PatchContext.cs | 1 + IllusionInjector/IllusionInjector.csproj | 4 +- .../Updating/ModsaberML/ApiEndpoint.cs | 28 ++- .../Updating/ModsaberML/Updater.cs | 164 ++++++++++++------ IllusionInjector/Utilities/BlockingStream.cs | 88 ---------- IllusionInjector/Utilities/EchoStream.cs | 40 ----- IllusionInjector/Utilities/LoneFunctions.cs | 42 +++++ ...ionInjector.csproj.CoreCompileInputs.cache | 2 +- 9 files changed, 181 insertions(+), 216 deletions(-) delete mode 100644 IllusionInjector/Utilities/BlockingStream.cs delete mode 100644 IllusionInjector/Utilities/EchoStream.cs create mode 100644 IllusionInjector/Utilities/LoneFunctions.cs diff --git a/IPA.Tests/updater_test.json b/IPA.Tests/updater_test.json index 36fca4a0..4f2ab017 100644 --- a/IPA.Tests/updater_test.json +++ b/IPA.Tests/updater_test.json @@ -15,32 +15,12 @@ "conflictsWith": [], "files": { "steam": { - "hash": "b38f5f58b0131e10d4676cc94dad5315e2999aa8", + "hash": "a94e7eea2f656b2830a86000ee222b6cb06f8da5", "files": { - "CustomSongs/One More Time/": "da39a3ee5e6b4b0d3255bfef95601890afd80709", - "CustomSongs/One More Time/cover.jpg": "95dbaecba1ac8165f4cd0a83950a31859931db27", - "CustomSongs/One More Time/Credits.txt": "d067a9f5e0e7c6301dd9f6e8e7a609e82815345c", - "CustomSongs/One More Time/Easy.json": "e93e234a6fd38afc3d216e6d0e1585f19cb49f69", - "CustomSongs/One More Time/Expert.json": "8bd7f7e0e0782dce7b77a3bc8bdbc091f2fd202f", - "CustomSongs/One More Time/Hard.json": "99e693c77d280c37a6cfd9f479f158cd6c84a354", - "CustomSongs/One More Time/info.json": "d214d1990af6debf38ccfdb7c57462926fb0995c", - "CustomSongs/One More Time/lyrics.srt": "b3ed8a5f4fc3595fb097943379fc8ea22548bd51", - "CustomSongs/One More Time/Normal.json": "0f7c68b29dfb8c5cfde34b0dc3238353e2f9b24b", - "CustomSongs/One More Time/One More Time.ogg": "4d74e9a58341439299512dce2155c97a00d3fb3b", - "IPA.exe": "57521373f240845798bb99723ceb19ee09774131", - "IPA/Backups/": "da39a3ee5e6b4b0d3255bfef95601890afd80709", - "IPA/Data/": "da39a3ee5e6b4b0d3255bfef95601890afd80709", - "IPA/Data/Managed/": "da39a3ee5e6b4b0d3255bfef95601890afd80709", - "IPA/Data/Managed/IllusionInjector.dll": "b7f05f3c7abc052fab29787e97dfa5533ce8ff7d", - "IPA/Data/Managed/IllusionInjector.pdb": "a4ca3e2aada545eae600a6b8971ca892682b6693", - "IPA/Data/Managed/IllusionPlugin.dll": "c715be1506e16367c6cdb1c8e7b8bba845233f0f", - "IPA/Data/Managed/IllusionPlugin.pdb": "516f73cc420bfcdcd38a991c7664d62d7deb745a", - "IPA/Data/Managed/IllusionPlugin.xml": "e79c7cd5bc5369056d262fd963451e4fb22f8021", - "IPA/Launcher.exe": "76a68378dd0ef1fe660b87b96b18c5e77709e3ff", - "Mono.Cecil.dll": "762fb07e4d81722f0a766460289c6114cd4b7dae", - "Plugins/SongLoaderPlugin.dll": "f4ab080fbcc5abc6005a868cd965920bac46ddaf" + "Plugins/": "da39a3ee5e6b4b0d3255bfef95601890afd80709", + "Plugins/RandomSong.dll": "64ee1ecfeda73c550004bdb5123c7ac47a45e428" }, - "url": "file://Z:/Users/aaron/Source/Repos/IPA-Reloaded-BeatSaber/IPA/bin/Debug/Debug.zip" + "url": "https://www.modsaber.ml/cdn/randomsong/1.0.0-default.zip" } }, "weight": 7, diff --git a/IPA/PatchContext.cs b/IPA/PatchContext.cs index 29f2508c..8a2ab808 100644 --- a/IPA/PatchContext.cs +++ b/IPA/PatchContext.cs @@ -20,6 +20,7 @@ namespace IPA public string DataPathDst { get; private set; } public string ManagedPath { get; private set; } public string EngineFile { get; private set; } + public string EngineWebRequestFile { get; private set; } public string AssemblyFile { get; private set; } public string[] Args { get; private set; } public string ProjectRoot { get; private set; } diff --git a/IllusionInjector/IllusionInjector.csproj b/IllusionInjector/IllusionInjector.csproj index be93c80d..d4a58b08 100644 --- a/IllusionInjector/IllusionInjector.csproj +++ b/IllusionInjector/IllusionInjector.csproj @@ -25,6 +25,7 @@ prompt 4 false + true none @@ -80,8 +81,7 @@ - - + diff --git a/IllusionInjector/Updating/ModsaberML/ApiEndpoint.cs b/IllusionInjector/Updating/ModsaberML/ApiEndpoint.cs index 8a48b8eb..94fdaa0f 100644 --- a/IllusionInjector/Updating/ModsaberML/ApiEndpoint.cs +++ b/IllusionInjector/Updating/ModsaberML/ApiEndpoint.cs @@ -1,4 +1,5 @@ -using SimpleJSON; +using IllusionInjector.Utilities; +using SimpleJSON; using System; using System.Collections.Generic; using System.Linq; @@ -25,8 +26,16 @@ namespace IllusionInjector.Updating.ModsaberML public string Title; public Version GameVersion; public string Author; - public string SteamFile = null; - public string OculusFile = null; + + public class PlatformFile + { + public byte[] Hash = new byte[20]; // 20 byte because sha1 is fucky + public Dictionary FileHashes = new Dictionary(); + public string DownloadPath = null; + } + + public PlatformFile SteamFile = null; + public PlatformFile OculusFile = null; public static Mod DecodeJSON(JSONObject obj) { @@ -43,10 +52,19 @@ namespace IllusionInjector.Updating.ModsaberML foreach (var item in obj["files"]) { var key = item.Key; + var pfile = new PlatformFile() + { + DownloadPath = item.Value["url"], + Hash = LoneFunctions.StringToByteArray(item.Value["hash"]) + }; + + foreach (var file in item.Value["files"]) + pfile.FileHashes.Add(file.Key, LoneFunctions.StringToByteArray(file.Value)); + if (key == "steam") - outp.SteamFile = item.Value["url"]; + outp.SteamFile = pfile; if (key == "oculus") - outp.OculusFile = item.Value["url"]; + outp.OculusFile = pfile; } return outp; diff --git a/IllusionInjector/Updating/ModsaberML/Updater.cs b/IllusionInjector/Updating/ModsaberML/Updater.cs index b430a22d..a2b2cdcf 100644 --- a/IllusionInjector/Updating/ModsaberML/Updater.cs +++ b/IllusionInjector/Updating/ModsaberML/Updater.cs @@ -6,6 +6,7 @@ using System.Collections; using System.Collections.Generic; using System.IO; using System.Linq; +using System.Security.Cryptography; using System.Text; using System.Text.RegularExpressions; using System.Threading; @@ -150,29 +151,29 @@ namespace IllusionInjector.Updating.ModsaberML StartCoroutine(UpdateModCoroutine(tempDirectory, item)); } } - - public class StreamDownloadHandler : DownloadHandlerScript + + class StreamDownloadHandler : DownloadHandlerScript { - public BlockingStream Stream { get; set; } - public StreamDownloadHandler(BlockingStream stream) + public MemoryStream Stream { get; set; } + + public StreamDownloadHandler(MemoryStream stream) : base() { Stream = stream; } - protected void ReceiveContentLength(long contentLength) + protected override void ReceiveContentLength(int contentLength) { - //(Stream.BaseStream as MemoryStream).Capacity = (int)contentLength; + Stream.Capacity = contentLength; Logger.log.Debug($"Got content length: {contentLength}"); } - protected void OnContentComplete() + protected override void CompleteContent() { - Stream.Open = false; Logger.log.Debug("Download complete"); } - protected bool ReceiveData(byte[] data, long dataLength) + protected override bool ReceiveData(byte[] data, int dataLength) { Logger.log.Debug("ReceiveData"); if (data == null || data.Length < 1) @@ -181,7 +182,7 @@ namespace IllusionInjector.Updating.ModsaberML return false; } - Stream.Write(data, 0, (int)dataLength); + Stream.Write(data, 0, dataLength); return true; } @@ -199,19 +200,54 @@ namespace IllusionInjector.Updating.ModsaberML } - private void DownloadPluginAsync(BlockingStream stream, UpdateStruct item, string tempdir) + private void ExtractPluginAsync(MemoryStream stream, UpdateStruct item, ApiEndpoint.Mod.PlatformFile fileInfo) { - Logger.log.Debug($"Getting ZIP file for {item.plugin.Plugin.Name}"); //var stream = await httpClient.GetStreamAsync(url); - using (var zipFile = new ZipInputStream(stream)) + var data = stream.GetBuffer(); + SHA1 sha = new SHA1CryptoServiceProvider(); + var hash = sha.ComputeHash(data); + if (!LoneFunctions.UnsafeCompare(hash, fileInfo.Hash)) + throw new Exception("The hash for the file doesn't match what is defined"); + + using (var zipFile = ZipFile.Read(stream)) { Logger.log.Debug("Streams opened"); - ZipEntry entry; - while ((entry = zipFile.GetNextEntry()) != null) + foreach (var entry in zipFile) { Logger.log.Debug(entry?.FileName ?? "NULL"); + + if (entry.IsDirectory) + { + Logger.log.Debug($"Creating directory {entry.FileName}"); + Directory.CreateDirectory(Path.Combine(Environment.CurrentDirectory, entry.FileName)); + } + else + { + using (var ostream = new MemoryStream((int)entry.UncompressedSize)) + { + entry.Extract(ostream); + ostream.Seek(0, SeekOrigin.Begin); + + sha = new SHA1CryptoServiceProvider(); + var fileHash = sha.ComputeHash(ostream); + if (!LoneFunctions.UnsafeCompare(fileHash, fileInfo.FileHashes[entry.FileName])) + throw new Exception("The hash for the file doesn't match what is defined"); + + ostream.Seek(0, SeekOrigin.Begin); + FileInfo targetFile = new FileInfo(Path.Combine(Environment.CurrentDirectory, entry.FileName)); + if (targetFile.Exists) + { + Logger.log.Debug($"Target file {targetFile.FullName} exists"); + } + + var fstream = targetFile.Create(); + ostream.CopyTo(fstream); + + Logger.log.Debug($"Wrote file {targetFile.FullName}"); + } + } } } @@ -220,59 +256,75 @@ namespace IllusionInjector.Updating.ModsaberML IEnumerator UpdateModCoroutine(string tempdir, UpdateStruct item) { - - string url; + ApiEndpoint.Mod.PlatformFile platformFile; if (SteamCheck.IsAvailable || item.externInfo.OculusFile == null) - url = item.externInfo.SteamFile; + platformFile = item.externInfo.SteamFile; else - url = item.externInfo.OculusFile; + platformFile = item.externInfo.OculusFile; + + string url = platformFile.DownloadPath; Logger.log.Debug($"URL = {url}"); - - using (var memStream = new EchoStream()) - using (var stream = new BlockingStream(memStream)) - using (var request = UnityWebRequest.Get(url)) - using (var taskTokenSource = new CancellationTokenSource()) + + const int MaxTries = 3; + int maxTries = MaxTries; + while (maxTries > 0) { - var dlh = new StreamDownloadHandler(stream); - request.downloadHandler = dlh; + if (maxTries-- != MaxTries) + Logger.log.Info($"Re-trying download..."); - var downloadTask = Task.Run(() => - { // use slightly more multithreaded approach than coroutines - DownloadPluginAsync(stream, item, tempdir); - }, taskTokenSource.Token); + using (var stream = new MemoryStream()) + using (var request = UnityWebRequest.Get(url)) + using (var taskTokenSource = new CancellationTokenSource()) + { + var dlh = new StreamDownloadHandler(stream); + request.downloadHandler = dlh; - Logger.log.Debug("Sending request"); - Logger.log.Debug(request?.downloadHandler?.ToString() ?? "DLH==NULL"); - yield return request.SendWebRequest(); - Logger.log.Debug("Download finished"); + Logger.log.Debug("Sending request"); + //Logger.log.Debug(request?.downloadHandler?.ToString() ?? "DLH==NULL"); + yield return request.SendWebRequest(); + Logger.log.Debug("Download finished"); - if (stream.Open) - { // anti-hang - Logger.log.Warn("Downloader failed to call DownloadHandler"); - stream.Open = false; // no more writing - stream.BaseStream.Write(new byte[] { 0 }, 0, 1); - } + if (request.isNetworkError) + { + Logger.log.Error("Network error while trying to update mod"); + Logger.log.Error(request.error); + taskTokenSource.Cancel(); + continue; + } + if (request.isHttpError) + { + Logger.log.Error($"Server returned an error code while trying to update mod"); + Logger.log.Error(request.error); + taskTokenSource.Cancel(); + continue; + } - if (request.isNetworkError) - { - Logger.log.Error("Network error while trying to update mod"); - Logger.log.Error(request.error); - taskTokenSource.Cancel(); - yield break; - } - if (request.isHttpError) - { - Logger.log.Error($"Server returned an error code while trying to update mod"); - Logger.log.Error(request.error); - taskTokenSource.Cancel(); - yield break; - } + stream.Seek(0, SeekOrigin.Begin); // reset to beginning - downloadTask.Wait(); // wait for the damn thing to finish + var downloadTask = Task.Run(() => + { // use slightly more multithreaded approach than coroutines + ExtractPluginAsync(stream, item, platformFile); + }, taskTokenSource.Token); + + while (!(downloadTask.IsCompleted || downloadTask.IsCanceled || downloadTask.IsFaulted)) + yield return null; // pause coroutine until task is done + + if (downloadTask.IsFaulted) + { + Logger.log.Error($"Error downloading mod {item.plugin.Plugin.Name}"); + Logger.log.Error(downloadTask.Exception); + continue; + } + + break; + } } - yield return null; + if (maxTries == 0) + Logger.log.Warn($"Plugin download failed {MaxTries} times, not re-trying"); + else + Logger.log.Debug("Download complete"); } } } diff --git a/IllusionInjector/Utilities/BlockingStream.cs b/IllusionInjector/Utilities/BlockingStream.cs deleted file mode 100644 index e23811bb..00000000 --- a/IllusionInjector/Utilities/BlockingStream.cs +++ /dev/null @@ -1,88 +0,0 @@ -using IllusionInjector.Logging; -using System; -using System.Collections.Generic; -using System.Diagnostics; -using System.IO; -using System.Linq; -using System.Text; -using System.Threading.Tasks; - -namespace IllusionInjector.Utilities -{ - class BlockingStream : Stream - { - public BlockingStream(Stream bstr) - { - BaseStream = bstr; - } - - public Stream BaseStream { get; set; } - - private bool _open = true; - public bool Open { - get - { - return CanWrite; - } - set - { - if (!_open) - throw new InvalidOperationException("Blocking stream has already been closed!"); - else - _open = value; - } - } - - private bool canReadOverride = true; - public override bool CanRead => BaseStream.CanRead && canReadOverride; - - public override bool CanSeek => BaseStream.CanSeek; - - public override bool CanWrite => BaseStream.CanWrite && _open; - - public override long Length => BaseStream.Length; - - public override long Position { get => BaseStream.Position; set => BaseStream.Position = value; } - - public override void Flush() - { - BaseStream.Flush(); - } - - public override int Read(byte[] buffer, int offset, int count) - { - var read = 0; - while (read < count && Open) - { - read += BaseStream.Read(buffer, read, count-read); - } - - if (read == 0) - { - canReadOverride = false; - } - - return read; - } - - public override long Seek(long offset, SeekOrigin origin) - { - return BaseStream.Seek(offset, origin); - } - - public override void SetLength(long value) - { - BaseStream.SetLength(value); - } - - public override void Write(byte[] buffer, int offset, int count) - { - BaseStream.Write(buffer, offset, count); - } - - public override string ToString() - { - return $"{base.ToString()} ({BaseStream?.ToString()})"; - } - } -} diff --git a/IllusionInjector/Utilities/EchoStream.cs b/IllusionInjector/Utilities/EchoStream.cs deleted file mode 100644 index 6d1a3ca0..00000000 --- a/IllusionInjector/Utilities/EchoStream.cs +++ /dev/null @@ -1,40 +0,0 @@ -using System; -using System.Collections.Generic; -using System.IO; -using System.Linq; -using System.Text; -using System.Threading; -using System.Threading.Tasks; - -namespace IllusionInjector.Utilities -{ - public class EchoStream : MemoryStream - { - private ManualResetEvent m_dataReady = new ManualResetEvent(false); - private byte[] m_buffer; - private int m_offset; - private int m_count; - - public override void Write(byte[] buffer, int offset, int count) - { - m_buffer = buffer; - m_offset = offset; - m_count = count; - m_dataReady.Set(); - } - - public override int Read(byte[] buffer, int offset, int count) - { - if (m_buffer == null) - { - // Block until the stream has some more data. - m_dataReady.Reset(); - m_dataReady.WaitOne(); - } - - Buffer.BlockCopy(m_buffer, m_offset, buffer, offset, (count < m_count) ? count : m_count); - m_buffer = null; - return (count < m_count) ? count : m_count; - } - } -} diff --git a/IllusionInjector/Utilities/LoneFunctions.cs b/IllusionInjector/Utilities/LoneFunctions.cs new file mode 100644 index 00000000..1dad987f --- /dev/null +++ b/IllusionInjector/Utilities/LoneFunctions.cs @@ -0,0 +1,42 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace IllusionInjector.Utilities +{ + public static class LoneFunctions + { + public static byte[] StringToByteArray(string hex) + { + int NumberChars = hex.Length; + byte[] bytes = new byte[NumberChars / 2]; + for (int i = 0; i < NumberChars; i += 2) + bytes[i / 2] = Convert.ToByte(hex.Substring(i, 2), 16); + return bytes; + } + + // Copyright (c) 2008-2013 Hafthor Stefansson + // Distributed under the MIT/X11 software license + // Ref: http://www.opensource.org/licenses/mit-license.php. + // From: https://stackoverflow.com/a/8808245/3117125 + public static unsafe bool UnsafeCompare(byte[] a1, byte[] a2) + { + if (a1 == a2) return true; + if (a1 == null || a2 == null || a1.Length != a2.Length) + return false; + fixed (byte* p1 = a1, p2 = a2) + { + byte* x1 = p1, x2 = p2; + int l = a1.Length; + for (int i = 0; i < l / 8; i++, x1 += 8, x2 += 8) + if (*((long*)x1) != *((long*)x2)) return false; + if ((l & 4) != 0) { if (*((int*)x1) != *((int*)x2)) return false; x1 += 4; x2 += 4; } + if ((l & 2) != 0) { if (*((short*)x1) != *((short*)x2)) return false; x1 += 2; x2 += 2; } + if ((l & 1) != 0) if (*((byte*)x1) != *((byte*)x2)) return false; + return true; + } + } + } +} diff --git a/IllusionInjector/obj/Debug/IllusionInjector.csproj.CoreCompileInputs.cache b/IllusionInjector/obj/Debug/IllusionInjector.csproj.CoreCompileInputs.cache index c8be9064..8446ea99 100644 --- a/IllusionInjector/obj/Debug/IllusionInjector.csproj.CoreCompileInputs.cache +++ b/IllusionInjector/obj/Debug/IllusionInjector.csproj.CoreCompileInputs.cache @@ -1 +1 @@ -c0ee92dfbaba38e571471f2b468aeb2a80b0f76b +53986b9c1e3d7933198e1551a3ec82914a34cbb2