From 76e1d8a497d19b6addcd1fbe17dd73d464fb663a Mon Sep 17 00:00:00 2001 From: Nils Andresen Date: Sat, 20 Apr 2019 20:38:05 +0200 Subject: [PATCH] new option --no-parallel to fix issue #1242 --- doc/commands/clone.md | 1 + doc/commands/create.md | 1 + doc/commands/init.md | 2 + doc/config.md | 4 ++ src/GitTfs.VsCommon/Wrappers.cs | 29 +++++------ src/GitTfs.VsFake/TfsHelper.VsFake.cs | 18 +++---- src/GitTfs/Commands/RemoteOptions.cs | 5 +- src/GitTfs/Core/Ext.cs | 50 ++++++++++++------- src/GitTfs/Core/RemoteConfigConverter.cs | 7 ++- src/GitTfs/Core/RemoteInfo.cs | 9 ++-- src/GitTfs/Core/TfsInterop/IWorkspace.cs | 6 +-- src/GitTfs/Core/TfsWorkspace.cs | 12 ++--- .../Core/RemoteConfigConverterTests.cs | 21 +++++--- src/GitTfsTest/Integration/InitTests.cs | 9 ++++ 14 files changed, 110 insertions(+), 64 deletions(-) diff --git a/doc/commands/clone.md b/doc/commands/clone.md index 9c77c10b..d3e0726f 100644 --- a/doc/commands/clone.md +++ b/doc/commands/clone.md @@ -26,6 +26,7 @@ a TFS source tree and fetch all the changesets --except-regex=VALUE a regex of exceptions to ignore-regex -u, --username=VALUE TFS username -p, --password=VALUE TFS password + --no-parallel disable parallel access of the TFS server --all, --fetch-all --parents --authors=VALUE Path to an Authors file to map TFS users to Git users diff --git a/doc/commands/create.md b/doc/commands/create.md index dde55e15..952daefe 100644 --- a/doc/commands/create.md +++ b/doc/commands/create.md @@ -30,6 +30,7 @@ Prefer the [clone](clone.md) or [init](init.md) command if the project folder al --except-regex=VALUE a regex of exceptions to ingore-regex -u, --username=VALUE TFS username -p, --password=VALUE TFS password + --no-parallel disable parallel access of the TFS server --all, --fetch-all --parents --authors=VALUE Path to an Authors file to map TFS users to Git diff --git a/doc/commands/init.md b/doc/commands/init.md index 99bf7671..3b2de052 100644 --- a/doc/commands/init.md +++ b/doc/commands/init.md @@ -26,8 +26,10 @@ Prefer the [clone](clone.md) command to initialize and fetch changesets from a T messages Use this when you're exporting from TFS and don't need to put data back into TFS. + --no-parallel disable parallel access of the TFS server -u, --username=VALUE TFS username -p, --password=VALUE TFS password + --no-parallel disable parallel access of the TFS server ## Examples diff --git a/doc/config.md b/doc/config.md index 215cefb1..7cd5bd62 100644 --- a/doc/config.md +++ b/doc/config.md @@ -65,3 +65,7 @@ key for the remote `default` is `tfs-remote.default.url`. can be set to `true` to make git-tfs create a tag for each TFS commit. This is disabled by default, because creating a lot of tags will slow down your git operations. +* `noparallel` + can be set to `true` to make disable parallel access to the TFS. + This can be useful in cases where the TFS has problems with + parallel access and reports `TF400030`. (See issue #1242) diff --git a/src/GitTfs.VsCommon/Wrappers.cs b/src/GitTfs.VsCommon/Wrappers.cs index a3b8b8c8..10c1a739 100644 --- a/src/GitTfs.VsCommon/Wrappers.cs +++ b/src/GitTfs.VsCommon/Wrappers.cs @@ -1,14 +1,14 @@ +using GitTfs.Core; +using GitTfs.Core.TfsInterop; +using GitTfs.Util; +using Microsoft.TeamFoundation.Server; +using Microsoft.TeamFoundation.VersionControl.Client; +using Microsoft.TeamFoundation.VersionControl.Common; using System; using System.Collections.Generic; using System.Diagnostics; using System.IO; using System.Linq; -using Microsoft.TeamFoundation.Server; -using Microsoft.TeamFoundation.VersionControl.Client; -using GitTfs.Core; -using GitTfs.Core.TfsInterop; -using GitTfs.Util; -using Microsoft.TeamFoundation.VersionControl.Common; namespace GitTfs.VsCommon { @@ -469,20 +469,20 @@ namespace GitTfs.VsCommon Retry.Do(() => DoUntilNoFailures(() => _workspace.Get(new ChangesetVersionSpec(changeset), GetOptions.Overwrite | GetOptions.GetAll))); } - public void GetSpecificVersion(int changesetId, IEnumerable items) + public void GetSpecificVersion(int changesetId, IEnumerable items, bool noParallel) { var version = new ChangesetVersionSpec(changesetId); - GetRequests(items.Select(e => new GetRequest(new ItemSpec(e.ServerItem, RecursionType.Full), version))); + GetRequests(items.Select(e => new GetRequest(new ItemSpec(e.ServerItem, RecursionType.Full), version)), noParallel); } - public void GetSpecificVersion(IChangeset changeset) + public void GetSpecificVersion(IChangeset changeset, bool noParallel) { - GetSpecificVersion(changeset.ChangesetId, changeset.Changes); + GetSpecificVersion(changeset.ChangesetId, changeset.Changes, noParallel); } - public void GetSpecificVersion(int changesetId, IEnumerable changes) + public void GetSpecificVersion(int changesetId, IEnumerable changes, bool noParallel) { - GetRequests(changes.Select(change => new GetRequest(new ItemSpec(change.Item.ServerItem, RecursionType.None, change.Item.DeletionId), changesetId))); + GetRequests(changes.Select(change => new GetRequest(new ItemSpec(change.Item.ServerItem, RecursionType.None, change.Item.DeletionId), changesetId)), noParallel); } public string GetLocalItemForServerItem(string serverItem) @@ -536,8 +536,9 @@ namespace GitTfs.VsCommon } } - public void GetRequests(IEnumerable source, int batchSize = 20) + public void GetRequests(IEnumerable source, bool noParallel, int batchSize = 20) { + source.ToBatch(batchSize).DoParallel(batch => { var items = batch; @@ -554,7 +555,7 @@ namespace GitTfs.VsCommon items = status.GetFailures().Join(items, e => e.ServerItem, e => e.ItemSpec.Item, (failure, request) => request).ToArray(); } }); - }); + }, noParallel); } } diff --git a/src/GitTfs.VsFake/TfsHelper.VsFake.cs b/src/GitTfs.VsFake/TfsHelper.VsFake.cs index 464d8440..edda0f4a 100644 --- a/src/GitTfs.VsFake/TfsHelper.VsFake.cs +++ b/src/GitTfs.VsFake/TfsHelper.VsFake.cs @@ -1,13 +1,13 @@ -using System; -using System.Collections.Generic; -using System.Diagnostics; -using System.IO; -using System.Linq; using GitTfs.Commands; using GitTfs.Core; using GitTfs.Core.TfsInterop; using GitTfs.Util; using StructureMap; +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.IO; +using System.Linq; namespace GitTfs.VsFake { @@ -258,17 +258,17 @@ namespace GitTfs.VsFake _repositoryRoot = repositoryRoot; } - public void GetSpecificVersion(int changesetId, IEnumerable items) + public void GetSpecificVersion(int changesetId, IEnumerable items, bool noParallel) { throw new NotImplementedException(); } - public void GetSpecificVersion(IChangeset changeset) + public void GetSpecificVersion(IChangeset changeset, bool noParallel) { - GetSpecificVersion(changeset.ChangesetId, changeset.Changes); + GetSpecificVersion(changeset.ChangesetId, changeset.Changes, noParallel); } - public void GetSpecificVersion(int changeset, IEnumerable changes) + public void GetSpecificVersion(int changeset, IEnumerable changes, bool noParallel) { var repositoryRoot = _repositoryRoot.ToLower(); if (!repositoryRoot.EndsWith("/")) repositoryRoot += "/"; diff --git a/src/GitTfs/Commands/RemoteOptions.cs b/src/GitTfs/Commands/RemoteOptions.cs index 6976b38e..f4eace02 100644 --- a/src/GitTfs/Commands/RemoteOptions.cs +++ b/src/GitTfs/Commands/RemoteOptions.cs @@ -1,5 +1,5 @@ -using NDesk.Options; using GitTfs.Util; +using NDesk.Options; namespace GitTfs.Commands { @@ -20,6 +20,8 @@ namespace GitTfs.Commands v => Username = v }, { "p|password=", "TFS password", v => Password = v }, + { "no-parallel", "Do not access tfs in parallel", + v => NoParallel = (v != null) }, }; } } @@ -28,5 +30,6 @@ namespace GitTfs.Commands public string ExceptRegex { get; set; } public string Username { get; set; } public string Password { get; set; } + public bool NoParallel { get; set; } } } diff --git a/src/GitTfs/Core/Ext.cs b/src/GitTfs/Core/Ext.cs index 355e8dbf..9f08213a 100644 --- a/src/GitTfs/Core/Ext.cs +++ b/src/GitTfs/Core/Ext.cs @@ -1,3 +1,6 @@ +using GitTfs.Commands; +using NDesk.Options; +using StructureMap; using System; using System.Collections.Generic; using System.Diagnostics; @@ -5,9 +8,6 @@ using System.IO; using System.Linq; using System.Text; using System.Text.RegularExpressions; -using NDesk.Options; -using GitTfs.Commands; -using StructureMap; namespace GitTfs.Core { @@ -175,7 +175,7 @@ namespace GitTfs.Core public static IEnumerable ToBatch(this IEnumerable source, Func selector, int batchSize) { var batch = new List(batchSize); - + foreach (var item in source) { if (batch.Count >= batchSize) @@ -211,17 +211,24 @@ namespace GitTfs.Core /// /// The source sequence. /// The action to execute. + /// set to true> to disable parallel processing /// The source item type - public static void DoParallel(this IEnumerable source, Action action) + public static void DoParallel(this IEnumerable source, Action action, bool noParallel) { #if DEBUG && NO_PARALLEL - foreach (var item in source) - { - action(item); - } -#else - (source as ParallelQuery ?? source.AsParallel()).ForAll(action); + noParallel = true; #endif + if (noParallel) + { + foreach (var item in source) + { + action(item); + } + } + else + { + (source as ParallelQuery ?? source.AsParallel()).ForAll(action); + } } /// @@ -236,15 +243,17 @@ namespace GitTfs.Core /// /// The delay between retries. /// + /// set to true to disable parallel processing /// The type of items in the . /// /// Returns true when all items processed successfully. public static void DoParallelRetry( this IEnumerable source, Action action, - TimeSpan retryInterval) + TimeSpan retryInterval, + bool noParallel) { - DoParallelRetry(source, action, 10, retryInterval); + DoParallelRetry(source, action, 10, retryInterval, noParallel); } /// @@ -256,12 +265,13 @@ namespace GitTfs.Core /// /// The action to process of the item. /// + /// set to true to disable parallel processing /// The type of items in the . /// /// Returns true when all items processed successfully. - public static void DoParallelRetry(this IEnumerable source, Action action) + public static void DoParallelRetry(this IEnumerable source, Action action, bool noParallel) { - DoParallelRetry(source, action, 10, TimeSpan.FromSeconds(1)); + DoParallelRetry(source, action, 10, TimeSpan.FromSeconds(1), noParallel); } /// @@ -276,12 +286,13 @@ namespace GitTfs.Core /// /// TThe number of retries. /// + /// set to true to disable parallel processing /// The type of items in the . /// /// Returns true when all items processed successfully. - public static void DoParallelRetry(this IEnumerable source, Action action, int retryCount) + public static void DoParallelRetry(this IEnumerable source, Action action, int retryCount, bool noParallel) { - DoParallelRetry(source, action, retryCount, TimeSpan.FromSeconds(1)); + DoParallelRetry(source, action, retryCount, TimeSpan.FromSeconds(1), noParallel); } /// @@ -299,10 +310,11 @@ namespace GitTfs.Core /// /// The delay between retries. /// + /// set to true to disable parallel processing /// The type of items in the . /// /// Returns true when all items processed successfully. - public static void DoParallelRetry(this IEnumerable source, Action action, int retryCount, TimeSpan retryInterval) + public static void DoParallelRetry(this IEnumerable source, Action action, int retryCount, TimeSpan retryInterval, bool noParallel) { var fails = new ConcurrentBag(); @@ -340,7 +352,7 @@ namespace GitTfs.Core { fails.Add(new AggregateException(exceptions)); } - }); + }, noParallel); if (fails.Count > 0) { diff --git a/src/GitTfs/Core/RemoteConfigConverter.cs b/src/GitTfs/Core/RemoteConfigConverter.cs index ebd1ebd4..f264635a 100644 --- a/src/GitTfs/Core/RemoteConfigConverter.cs +++ b/src/GitTfs/Core/RemoteConfigConverter.cs @@ -1,6 +1,6 @@ -using System.Collections.Generic; +using LibGit2Sharp; +using System.Collections.Generic; using System.Linq; -using LibGit2Sharp; namespace GitTfs.Core { @@ -35,6 +35,8 @@ namespace GitTfs.Core remote.Aliases = entry.Value.Split(','); else if (key == "autotag") remote.Autotag = bool.Parse(entry.Value); + else if (key == "noparallel") + remote.NoParallel = bool.Parse(entry.Value); } } return remotes.Values.Where(r => !string.IsNullOrWhiteSpace(r.Url)); @@ -53,6 +55,7 @@ namespace GitTfs.Core yield return c(prefix + "ignore-except", remote.IgnoreExceptRegex); yield return c(prefix + "legacy-urls", remote.Aliases == null ? null : string.Join(",", remote.Aliases)); yield return c(prefix + "autotag", remote.Autotag ? "true" : null); + yield return c(prefix + "noparallel", remote.NoParallel ? "true" : null); } } diff --git a/src/GitTfs/Core/RemoteInfo.cs b/src/GitTfs/Core/RemoteInfo.cs index b3df3944..f0309a1c 100644 --- a/src/GitTfs/Core/RemoteInfo.cs +++ b/src/GitTfs/Core/RemoteInfo.cs @@ -1,5 +1,5 @@ -using System.Collections.Generic; -using GitTfs.Commands; +using GitTfs.Commands; +using System.Collections.Generic; namespace GitTfs.Core { @@ -14,11 +14,12 @@ namespace GitTfs.Core public string IgnoreExceptRegex { get; set; } public IEnumerable Aliases { get; set; } public bool Autotag { get; set; } + public bool NoParallel { get; set; } public RemoteOptions RemoteOptions { - get { return new RemoteOptions { IgnoreRegex = IgnoreRegex, ExceptRegex = IgnoreExceptRegex, Username = Username, Password = Password }; } - set { IgnoreRegex = value.IgnoreRegex; IgnoreExceptRegex = value.ExceptRegex; Username = value.Username; Password = value.Password; } + get { return new RemoteOptions { IgnoreRegex = IgnoreRegex, ExceptRegex = IgnoreExceptRegex, Username = Username, Password = Password, NoParallel = NoParallel }; } + set { IgnoreRegex = value.IgnoreRegex; IgnoreExceptRegex = value.ExceptRegex; Username = value.Username; Password = value.Password; NoParallel = value.NoParallel; } } } } diff --git a/src/GitTfs/Core/TfsInterop/IWorkspace.cs b/src/GitTfs/Core/TfsInterop/IWorkspace.cs index 1fb29332..e8b1592c 100644 --- a/src/GitTfs/Core/TfsInterop/IWorkspace.cs +++ b/src/GitTfs/Core/TfsInterop/IWorkspace.cs @@ -14,9 +14,9 @@ namespace GitTfs.Core.TfsInterop int PendRename(string pathFrom, string pathTo); void ForceGetFile(string path, int changeset); void GetSpecificVersion(int changeset); - void GetSpecificVersion(int changeset, IEnumerable items); - void GetSpecificVersion(IChangeset changeset); - void GetSpecificVersion(int changeset, IEnumerable changes); + void GetSpecificVersion(int changeset, IEnumerable items, bool noParallel); + void GetSpecificVersion(IChangeset changeset, bool noParallel); + void GetSpecificVersion(int changeset, IEnumerable changes, bool noParallel); string GetLocalItemForServerItem(string serverItem); string GetServerItemForLocalItem(string localItem); string OwnerName { get; } diff --git a/src/GitTfs/Core/TfsWorkspace.cs b/src/GitTfs/Core/TfsWorkspace.cs index 7f1c5e79..68fe7d6d 100644 --- a/src/GitTfs/Core/TfsWorkspace.cs +++ b/src/GitTfs/Core/TfsWorkspace.cs @@ -1,11 +1,11 @@ +using GitTfs.Commands; +using GitTfs.Core.TfsInterop; using System; using System.Collections.Generic; using System.Collections.ObjectModel; +using System.Diagnostics; using System.IO; using System.Linq; -using GitTfs.Commands; -using GitTfs.Core.TfsInterop; -using System.Diagnostics; namespace GitTfs.Core { @@ -237,19 +237,19 @@ namespace GitTfs.Core public void Get(int changesetId, IEnumerable items) { - _workspace.GetSpecificVersion(changesetId, items); + _workspace.GetSpecificVersion(changesetId, items, Remote.RemoteInfo.NoParallel); } public void Get(IChangeset changeset) { - _workspace.GetSpecificVersion(changeset); + _workspace.GetSpecificVersion(changeset, Remote.RemoteInfo.NoParallel); } public void Get(int changesetId, IEnumerable changes) { if (changes.Any()) { - _workspace.GetSpecificVersion(changesetId, changes); + _workspace.GetSpecificVersion(changesetId, changes, Remote.RemoteInfo.NoParallel); } } diff --git a/src/GitTfsTest/Core/RemoteConfigConverterTests.cs b/src/GitTfsTest/Core/RemoteConfigConverterTests.cs index 1f013349..483584da 100644 --- a/src/GitTfsTest/Core/RemoteConfigConverterTests.cs +++ b/src/GitTfsTest/Core/RemoteConfigConverterTests.cs @@ -1,9 +1,9 @@ -using System.Collections.Generic; -using System.Linq; -using Xunit; +using GitTfs.Commands; using GitTfs.Core; using LibGit2Sharp; -using GitTfs.Commands; +using System.Collections.Generic; +using System.Linq; +using Xunit; namespace GitTfs.Test.Core { @@ -39,6 +39,7 @@ namespace GitTfs.Test.Core AssertContainsConfig("tfs-remote.default.ignore-paths", null, config); AssertContainsConfig("tfs-remote.default.legacy-urls", null, config); AssertContainsConfig("tfs-remote.default.autotag", null, config); + AssertContainsConfig("tfs-remote.default.noparallel", null, config); } [Fact] @@ -55,6 +56,7 @@ namespace GitTfs.Test.Core IgnoreExceptRegex = "def", Autotag = true, Aliases = new string[] { "http://abc", "http://def" }, + NoParallel = true, }; var config = _dumper.Dump(remote); AssertContainsConfig("tfs-remote.default.url", "http://server/path", config); @@ -65,6 +67,7 @@ namespace GitTfs.Test.Core AssertContainsConfig("tfs-remote.default.ignore-except", "def", config); AssertContainsConfig("tfs-remote.default.legacy-urls", "http://abc,http://def", config); AssertContainsConfig("tfs-remote.default.autotag", "true", config); + AssertContainsConfig("tfs-remote.default.noparallel", "true", config); } /// @@ -83,7 +86,8 @@ namespace GitTfs.Test.Core Username = "user", Password = "pass", IgnoreRegex = "abc", - ExceptRegex = "def" + ExceptRegex = "def", + NoParallel = true }, Autotag = true, Aliases = new[] { "http://abc", "http://def" }, @@ -97,6 +101,7 @@ namespace GitTfs.Test.Core AssertContainsConfig("tfs-remote.default.ignore-except", "def", config); AssertContainsConfig("tfs-remote.default.legacy-urls", "http://abc,http://def", config); AssertContainsConfig("tfs-remote.default.autotag", "true", config); + AssertContainsConfig("tfs-remote.default.noparallel", "true", config); } /// @@ -116,6 +121,7 @@ namespace GitTfs.Test.Core IgnoreExceptRegex = "def", Autotag = true, Aliases = new[] { "http://abc", "http://def" }, + NoParallel = true }; var remoteOptions = remote.RemoteOptions; @@ -123,6 +129,7 @@ namespace GitTfs.Test.Core Assert.Equal("pass", remoteOptions.Password); Assert.Equal("abc", remoteOptions.IgnoreRegex); Assert.Equal("def", remoteOptions.ExceptRegex); + Assert.True(remoteOptions.NoParallel); } private void AssertContainsConfig(string key, string value, IEnumerable> configs) @@ -183,7 +190,8 @@ namespace GitTfs.Test.Core c("tfs-remote.default.ignore-paths", "ignorethis.zip"), c("tfs-remote.default.ignore-except", "dontignorethis.zip"), c("tfs-remote.default.legacy-urls", "http://old:8080/,http://other/"), - c("tfs-remote.default.autotag", "true")); + c("tfs-remote.default.autotag", "true"), + c("tfs-remote.default.noparallel", "true")); Assert.Single(remotes); var remote = remotes.First(); Assert.Equal("default", remote.Id); @@ -195,6 +203,7 @@ namespace GitTfs.Test.Core Assert.Equal("dontignorethis.zip", remote.IgnoreExceptRegex); Assert.Equal(new string[] { "http://old:8080/", "http://other/" }, remote.Aliases); Assert.True(remote.Autotag); + Assert.True(remote.NoParallel); } diff --git a/src/GitTfsTest/Integration/InitTests.cs b/src/GitTfsTest/Integration/InitTests.cs index b9232d14..b26ab7e3 100644 --- a/src/GitTfsTest/Integration/InitTests.cs +++ b/src/GitTfsTest/Integration/InitTests.cs @@ -45,5 +45,14 @@ namespace GitTfs.Test.Integration h.AssertRef("MyProject", "refs/remotes/tfs/default", expectedSha); h.AssertRef("MyProject", "refs/tags/tfs/default/C1", expectedSha); } + + [FactExceptOnUnix] + public void InitializesConfigUsingNoParallel() + { + h.SetupFake(r => { }); + h.Run("init", "http://my-tfs.local/tfs", "$/MyProject", "MyProject", "--no-parallel"); + h.AssertConfig("MyProject", "tfs-remote.default.noparallel", "true"); + } + } }