diff --git a/.gitignore b/.gitignore new file mode 100644 index 00000000..0175e4c1 --- /dev/null +++ b/.gitignore @@ -0,0 +1,2 @@ +.emacs-saves +*#* diff --git a/Makefile b/Makefile index 764bd656..961fca79 100644 --- a/Makefile +++ b/Makefile @@ -15,9 +15,17 @@ SHARE = share # Install variables: PREFIX ?= /usr/local -INSTALL_LIB ?= $(DESTDIR)$(shell git --exec-path) +GIT ?= git +INSTALL_LIB ?= $(shell $(GIT) --exec-path) INSTALL_EXT ?= $(INSTALL_LIB)/$(NAME).d -INSTALL_MAN1 ?= $(DESTDIR)$(PREFIX)/share/man/man1 +INSTALL_MAN1 ?= $(PREFIX)/share/man/man1 + +all: + echo $(INSTALL_LIB) + which git + git --exec-path + + # Basic targets: default: help diff --git a/README.fix-nested-subrepos.txt b/README.fix-nested-subrepos.txt new file mode 100644 index 00000000..550d2009 --- /dev/null +++ b/README.fix-nested-subrepos.txt @@ -0,0 +1,84 @@ +Background: +========== + Have a tree of ~270 nested subrepos of various levels + + top + | + subrepo0 -- top + | + import + / | \ + subreop1 subreop2 .. subrepo140 + | + import ... + / \ + sibreopo141 .. + + + checkout time for all files of the 'top' subrepo is about 20 min + +Issues found: +============ + + 1. 'subrepo branch' for the top of the nested subrepos creates a tree which includes nested subrepos and pushes them tot he + origin + + 2. 'subrepo bfanch' created nested references which just do not work in case of the multiple branches in existence. + e.g.: error: cannot lock ref 'refs/heads/subrepo/s1': 'refs/heads/subrepo/s1/s5/s6' exists; cannot create 'refs/heads/subrepo/s1' + + 3. '--tree-filter' used int the branch filtering is veeeery slow. My initial experinent of creating a branch for 'subrepo0' took + 52 hours. + + 4. 'multiple commands are missing the --ALL' qualifier + + +Bugs found: +========== + + 1. Worktree was not cleaned at subrepo:branch. It was causing issues in push --ALL if some branches already existed. + + 2. test/branch-rev-list-one-path.t failed internittently + + 3. encoding did not catch single '@' correclty, causing worktree creation to fail, at least in git 2.22 + +Features change: +=============== + + 1. git 2.22 message letter casing was changed from 'Couldn't find remote ref' to 'couldn't find remote ref'. + + +Added changes: +============= + 1. Added feature to clean nested subrepo in the filter-branch + + 2. flattened names of nested branches by replaceing '/' with '-' + + 3. Added extra checking for the updates done to the nested subrepos only. It checks updated files against the nested subrepo regex + -- found a code which created EMPTY commits, which looked like a leftower from debugging. It stayed in the way and I + commented it out. Did not affect any test. + -- it reduced number of revisions needed for subrepo0 in my initial case from 268 to 4 :-) + + 4. Replaced --tree-filter with --index-filter in soc:branch. For different subrepos performance was improved 2x to 10x. + -- found a git issue, probably related to the tree size. It crashed in filter-branch with `xrealloc(-1ULL)`. I did not + investigate it further. This happened to the top subrepo with 268 revisions. It worked with 4 (from above) and took onlly 7 + min to finish (vs 52 hours initially) + + -- aded the '--use_tree_filter' qualifier to allow old --tree-filter in case of git issues. + + 5. added the --squash_branch (-S) feature to the branch, push, pull, and fetch commands. It causes subrepo:branch to squash all commits into + one with combined log. This was initially done for performance reasons + + 6. fixed found bugs and updated features. + + -- added --ALL to 'branch', 'clean', 'fetch', 'pull', and 'push' + -- added --topo-order to 'rev-list' in subrepo:branch. This makes git reporting consistent and it looks like + branch-rev-list-one-path.t passes consistently now + -- claned 'wortree' in subrepo:branch before checking for existense fo the branch + -- added checking of non-prefixed branch names to the encoding. It was ok for branches but not ok for worktrees. Now + worktree seems to work. + -- fixed message regex to handle both, capitalized and non-capitalized veresion of the '[Cc]ouldn't' + + 7. added nested.t test to check both regular and squashed branches. + + + diff --git a/lib/git-subrepo b/lib/git-subrepo index 68918cbe..a641607d 100755 --- a/lib/git-subrepo +++ b/lib/git-subrepo @@ -1,4 +1,4 @@ -#!/usr/bin/env bash +#!/usr/bin/env bash # # # Copyright 2013-2017 - Ingy döt Net @@ -71,12 +71,17 @@ M,method= Method when you join, valid options are 'merge' or 'rebase' m,message= Specify a commit message r,remote= Specify the upstream remote to push/pull/fetch s,squash Squash commits on push +S,squash_branch Squash intermediate subrepo branch u,update Add the --branch and/or --remote overrides to .gitrepo q,quiet Show minimal output v,verbose Show verbose output d,debug Show the actual commands used x,DEBUG Turn on -x Bash debugging + +use_tree_filter subrepo branch normally uses 'index-filter' for performance reasons. + It is noted that this could hit some git limitations for huge trees and many revisions. + This qualifier is supposed to be workaround solution. " #------------------------------------------------------------------------------ @@ -94,6 +99,8 @@ main() { local force_wanted=false # Force certain operations local fetch_wanted=false # Fetch requested before a command local squash_wanted=false # Squash commits on push + local squash_branch_wanted=false # Squash subrepo brahcn + local use_tree_filter_wanted=false # use tree-filter instead of the index-filter. local update_wanted=false # Update .gitrepo with --branch and/or --remote local quiet_wanted=false # Output should be quiet @@ -143,6 +150,7 @@ main() { # Check environment and parse CLI options: assert-environment-ok + # Parse and validate command options: get-command-options "$@" @@ -265,6 +273,16 @@ command:fetch() { fi } +## +# srg: subrepo/$subref creates issues with nested subrepos. +# i'd like to flatten it. +## +subrepo-branch-name() { + local ref=${1:-$subref} + subref=${ref//\//-} + echo "subrepo/$subref" +} + # `git subrepo branch ` command: command:branch() { command-setup +subdir @@ -272,7 +290,9 @@ command:branch() { CALL subrepo:fetch fi - local branch="subrepo/$subref" + local branch=$(subrepo-branch-name) #"subrepo/$flatSubref" + o "srg: branch for '$subref' is '$branch'" + if $force_wanted; then # We must make sure that the worktree is removed as well worktree="$GIT_TMP/$branch" @@ -300,8 +320,9 @@ command:commit() { error "Can't find ref '$refs_subrepo_fetch'. Try using -F." upstream_head_commit="$(git rev-parse "$refs_subrepo_fetch")" + local subrepoBranchName=$(subrepo-branch-name) [[ -n $subrepo_commit_ref ]] || - subrepo_commit_ref="subrepo/$subref" + subrepo_commit_ref="$subrepoBranchName" subrepo:commit say "Subrepo commit '$subrepo_commit_ref' committed as" @@ -315,11 +336,12 @@ command:status() { status-refs() { local output= + local subrepoBranchName=$(subrepo-branch-name) while read line; do - [[ $line =~ ^([0-9a-f]+)\ refs/subrepo/$subref/([a-z]+) ]] || continue + [[ $line =~ ^([0-9a-f]+)\ refs/$subrepoBranchName/([a-z]+) ]] || continue local sha1=; sha1="$(git rev-parse --short "${BASH_REMATCH[1]}")" local type="${BASH_REMATCH[2]}" - local ref="refs/subrepo/$subref/$type" + local ref="refs/$subrepoBranchName/$type" if [[ $type == branch ]]; then output+=" Branch Ref: $sha1 ($ref)"$'\n' elif [[ $type == commit ]]; then @@ -528,7 +550,7 @@ subrepo:pull() { OK=false; CODE=-1; return fi - local branch_name="subrepo/$subref" + local branch_name=$(subrepo-branch-name) git:delete-branch "$branch_name" subrepo_commit_ref="$branch_name" @@ -575,13 +597,16 @@ subrepo:push() { local new_upstream=false local branch_created=false + o "srg: pusing $branch_name" + if [[ -z $branch_name ]]; then FAIL=false OUT=false CALL subrepo:fetch if ! OK; then # Check if we are pushing to a new upstream repo (or branch) and just # push the commit directly. This is common after a `git subrepo init`: - local re="(^|"$'\n'")fatal: Couldn't find remote ref " + # git 2.22 changed casing in the message to 'could' instead of 'Could': + local re="(^|"$'\n'")fatal: [Cc]ouldn't find remote ref " if [[ $output =~ $re ]]; then o "Pushing to new upstream: $subrepo_remote ($subrepo_branch)." new_upstream=true @@ -598,7 +623,7 @@ subrepo:push() { fi fi - branch_name="subrepo/$subref" + branch_name=$(subrepo-branch-name) git:delete-branch "$branch_name" if $squash_wanted; then @@ -646,8 +671,7 @@ subrepo:push() { if ! $force_wanted; then o "Make sure '$branch_name' contains the '$refs_subrepo_fetch' HEAD." if ! git:commit-in-rev-list "$upstream_head_commit" "$branch_name"; then - error "Can't commit: '$branch_name' doesn't contain upstream HEAD: " \ - "$upstream_head_commit" + error "Can't commit: '$branch_name' doesn't contain upstream HEAD: $upstream_head_commit" fi fi @@ -672,6 +696,7 @@ subrepo:push() { RUN git commit -m "$(get-commit-message)" } + # Fetch the subrepo's remote branch content: subrepo:fetch() { if [[ $subrepo_remote == none ]]; then @@ -690,21 +715,69 @@ subrepo:fetch() { git:make-ref "$refs_subrepo_fetch" FETCH_HEAD^0 } -# Create a subrepo branch containing all changes +# +# creates file path and returns an absolute path for the file +# +create-file-path() { + local dirname=$(dirname $1) + local basename=$(basename $1) + mkdir -p $dirname + + # realpath does not exist everywhere. this is a portable way. + local cwd=$PWD + cd $dirname + echo "$PWD/$basename" + cd $cwd +} + subrepo:branch() { - local branch="${1:-"subrepo/$subref"}" + local branch=$(subrepo-branch-name) + branch="${1:-$branch}" #"subrepo/$flatSubref"}" + + # clean up the worktree reference. It gets reused at least in 'push' + # and is not updated from the previouse -ALL branch if the current one exists. + worktree= + o "Check if the '$branch' branch already exists." git:branch-exists "$branch" && return + local curdir=$PWD + + ## + # srg: removing nested subrepos from the parent + ## + o 'Looking for nested subrepos' + get-all-subrepos "$subdir -mindepth 2" + local nested=(${subrepos[@]}) + local n= + local nestedSubrepos= + local nestedRegex= + for n in ${nested[@]}; do + [ -n "$nestedRegex" ] && nestedRegex+='|' + nestedRegex+="$n" + + n=${n#$subref/} + n=${n%/} + nestedSubrepos+=" $n" + done + + local msg_name=$(create-file-path "$GIT_TMP/${branch}.msg") + if $squash_branch_wanted ; then + echo "Squashed multipe commits" > $msg_name + fi + local last_gitrepo_commit= local first_gitrepo_commit= + local new_commit= + local author_info= o "Subrepo parent: $subrepo_parent" if [[ -n "$subrepo_parent" ]]; then local prev_commit= local ancestor= o "Create new commits with parents into the subrepo fetch" - OUT=true RUN git rev-list --reverse --ancestry-path "$subrepo_parent..HEAD" + + OUT=true RUN git rev-list --reverse --ancestry-path --topo-order "$subrepo_parent..HEAD" local commit_list="$output" for commit in $commit_list; do o "Working on $commit" @@ -741,51 +814,83 @@ subrepo:branch() { fi fi - o "Find parents" - local first_parent= - [[ -n $prev_commit ]] && first_parent="-p $prev_commit" - local second_parent= - if [[ -z "$first_gitrepo_commit" ]]; then - first_gitrepo_commit="$gitrepo_commit" - second_parent="-p $gitrepo_commit" - fi - - if [[ "$join_method" != "rebase" ]]; then - # In the rebase case we don't create merge commits - if [[ "$gitrepo_commit" != "$last_gitrepo_commit" ]]; then - second_parent="-p $gitrepo_commit" - last_gitrepo_commit="$gitrepo_commit" - fi - fi - - o "Create a new commit $first_parent $second_parent" - FAIL=false RUN git cat-file -e "$commit":"$subdir" - if OK; then - o "Create with content" - local PREVIOUS_IFS=$IFS - IFS=$'\n' - local author_info=( $(git log -1 --format=%ad%n%ae%n%an "$commit") ) - IFS=$PREVIOUS_IFS - - # When we create new commits we leave the author information unchanged - # the committer will though be updated to the current user - # This should be analog how cherrypicking is handled allowing git - # to store both the original author but also the responsible committer - # that created the local version of the commit and pushed it. - prev_commit=$(git log -n 1 --format=%B "$commit" | - GIT_AUTHOR_DATE="${author_info[0]}" \ - GIT_AUTHOR_EMAIL="${author_info[1]}" \ - GIT_AUTHOR_NAME="${author_info[2]}" \ - git commit-tree -F - $first_parent $second_parent "$commit":"$subdir") - else - o "Create empty placeholder" - prev_commit=$(git commit-tree -m "EMPTY" \ - $first_parent $second_parent "4b825dc642cb6eb9a060e54bf8d69288fbee4904") - fi + o "Find parents" + local first_parent= + [[ -n $prev_commit ]] && first_parent="-p $prev_commit" + local second_parent= + if [[ -z "$first_gitrepo_commit" ]]; then + first_gitrepo_commit="$gitrepo_commit" + second_parent="-p $gitrepo_commit" + fi + + if [[ "$join_method" != "rebase" ]]; then + # In the rebase case we don't create merge commits + if [[ "$gitrepo_commit" != "$last_gitrepo_commit" ]]; then + second_parent="-p $gitrepo_commit" + last_gitrepo_commit="$gitrepo_commit" + fi + fi + + o "Create a new commit $first_parent $second_parent" + + # it is real if: + # 1. the subdir *is* in the commit + # 2. there are files which do not belong to subrepos, but belong to the current one + # + FAIL=false RUN git cat-file -e "$commit":"$subdir" + + if OK && [ -n "$nestedRegex" ]; then + OUT=true RUN git log --name-only --pretty=format:'' $commit -1 + local files=$output + OK=false + for file in "$files"; do + if ! [[ $file =~ $nestedRegex ]] && [[ $file =~ $subdir ]] ; then + OK=true + break + fi + done + fi + + if OK ; then + o "Create with content" + local PREVIOUS_IFS=$IFS + IFS=$'\n' + author_info=( $(git log -1 --format=%ad%n%ae%n%an "$commit") ) + IFS=$PREVIOUS_IFS + + + # When we create new commits we leave the author information unchanged + # the committer will though be updated to the current user + # This should be analog how cherrypicking is handled allowing git + # to store both the original author but also the responsible committer + # that created the local version of the commit and pushed it. + prev_commit=$(git log -n 1 --format=%B "$commit" | + GIT_AUTHOR_DATE="${author_info[0]}" \ + GIT_AUTHOR_EMAIL="${author_info[1]}" \ + GIT_AUTHOR_NAME="${author_info[2]}" \ + git commit-tree -F - $first_parent $second_parent "$commit":"$subdir") + + if $squash_branch_wanted ; then + # add to the squash commit message + git log $commit -1 --pretty="%n===%H%nAuthor: %an%nEmail: %ae%nDate: %ad%n%n%B" >> $msg_name + fi + + else + o "Create empty placeholder" + + # + #srg: why do we need to create it? + # looks like a left-over from debug. It stays int he way of figuring out the set of non-empty commits. It is not get pruned later. + # + # prev_commit=$(git commit-tree -m "EMPTY" \ + # $first_parent $second_parent "4b825dc642cb6eb9a060e54bf8d69288fbee4904") + fi + done - o "Create branch '$branch' for this new commit set $prev_commit." - RUN git branch "$branch" "$prev_commit" + o "Create branch '$branch' for this new commit set $prev_commit" + RUN git branch "$branch" "$prev_commit" + else o "No parent setting, use the subdir content." RUN git branch "$branch" HEAD @@ -793,18 +898,49 @@ subrepo:branch() { "$subref" "$branch" fi - o "Remove the .gitrepo file from $first_gitrepo_commit..$branch" + + ## squashing + if $squash_branch_wanted && [ -n "$gitrepo_commit" ] && [[ $(git rev-list $branch | wc -l) > 1 ]]; then + o "Squashing branch $branch to $gitrepo_commit" + local squashedCommit=$(git commit-tree "$branch^{tree}" -F $msg_name -p $gitrepo_commit) + if [[ $squashedCommit =~ ^[[:xdigit:]]{40}$ ]]; then + RUN git update-ref "refs/heads/$branch" $squashedCommit + else + say "error: Squashing branch $branch to $gitrepo_commit. commit-tree did not produce a valid commit sha1. Got '$squashedCommit'." + # will continue and do non-smashed version. + fi + fi + + o "Remove the .gitrepo $nestedSubrepos files from $first_gitrepo_commit..$branch ( $(git rev-list $first_gitrepo_commit..$branch | wc -l) ) commits." local filter="$branch" [[ -n "$first_gitrepo_commit" ]] && filter="$first_gitrepo_commit..$branch" - FAIL=false RUN git filter-branch -f --prune-empty --tree-filter \ - "rm -f .gitrepo" "$filter" - git:create-worktree "$branch" + ## + # srg: There was an issue with git commit-index causing a git crash with 'xrealloc(-1ULL)' after checking about 96 revisions in filter-branch --index-filter. + # The thing worked with -tree-filter, but is way to slow. + # so, i decided to keep the tree fileter here and make it optional. + ## + + if $use_tree_filter_wanted ; then + o "Using --tree-filter" + FAIL=false RUN git filter-branch -f --prune-empty --tree-filter \ + "rm -rf .gitrepo $nestedSubrepos" "$filter" + else + FAIL=false RUN git filter-branch -f --prune-empty \ + --index-filter "git rm -rf .gitrepo $nestedSubrepos --cached --ignore-unmatch" \ + "$filter" + fi + + o "Creating worktree for $branch" + git:create-worktree "$branch" o "Create ref '$refs_subrepo_branch'." git:make-ref "$refs_subrepo_branch" "$branch" + + cd $curdir } + # Commit a merged subrepo branch: subrepo:commit() { o "Check that '$subrepo_commit_ref' exists." @@ -894,7 +1030,8 @@ subrepo:status() { continue fi - refs_subrepo_fetch="refs/subrepo/$subref/fetch" + local branchName=$(subrepo-branch-name) + refs_subrepo_fetch="refs/$branchName/fetch" upstream_head_commit="$( git rev-parse --short "$refs_subrepo_fetch" 2> /dev/null || true )" @@ -912,12 +1049,12 @@ subrepo:status() { fi echo "Git subrepo '$subdir':" - git:branch-exists "subrepo/$subref" && - echo " Subrepo Branch: subrepo/$subref" - local remote="subrepo/$subref" + git:branch-exists "$branchName" && + echo " Subrepo Branch: $branchName" + local remote="$branchName" FAIL=false OUT=true RUN git config "remote.$remote.url" [[ -n $output ]] && - echo " Remote Name: subrepo/$subref" + echo " Remote Name: $branchName" echo " Remote URL: $subrepo_remote" [[ -n $upstream_head_commit ]] && echo " Upstream Ref: $upstream_head_commit" @@ -933,7 +1070,8 @@ subrepo:status() { fi # Grep for directory, branch can be in detached state due to conflicts - local _worktree=$(git worktree list | grep "$GIT_TMP/subrepo/$subdir") + local branchName=$(subrepo-branch-name $subdir) + local _worktree=$(git worktree list | grep -P "$GIT_TMP/$branchName\s" ) #subrepo/$subdir") if [[ -n $_worktree ]]; then echo " Worktree: $_worktree" fi @@ -947,8 +1085,10 @@ subrepo:status() { } subrepo:clean() { + local branchName=$(subrepo-branch-name) + # Remove subrepo branches if exist: - local branch="subrepo/$subref" + local branch="$branchName" local ref="refs/heads/$branch" local worktree="$GIT_TMP/$branch" @@ -965,7 +1105,7 @@ subrepo:clean() { if "$all_wanted"; then RUN rm -fr .git/refs/subrepo/ else - RUN rm -fr .git/refs/subrepo/$subref/ + RUN rm -fr .git/refs/$branchName/ fi fi } @@ -987,7 +1127,7 @@ get-command-options() { [[ -n $GIT_SUBREPO_VERBOSE ]] && verbose_wanted=true [[ -n $GIT_SUBREPO_DEBUG ]] && debug_wanted=true - eval "$( + eval "$( echo "$GETOPT_SPEC" | git rev-parse --parseopt -- "$@" || echo exit $? @@ -1019,6 +1159,7 @@ get-command-options() { commit_msg_args+=("--remote=$1") shift ;; -s) squash_wanted=true ;; + -S) squash_branch_wanted=true ;; -u) update_wanted=true commit_msg_args+=("--update") ;; -q) quiet_wanted=true ;; @@ -1028,6 +1169,9 @@ get-command-options() { --version) echo "$VERSION" exit ;; + --use_tree_filter) + use_tree_filter_wanted=true + ;; *) usage-error "Unexpected option: '$option'." ;; esac done @@ -1047,7 +1191,7 @@ get-command-options() { fi commit_msg_args+=("${command_arguments[@]}") - for option in all ALL edit fetch force squash; do + for option in all ALL edit fetch force squash squash_branch use_tree_filter; do var="${option}_wanted" if ${!var}; then check_option $option @@ -1069,22 +1213,24 @@ get-command-options() { usage-error "Can't use '--update' without '--branch' or '--remote'." fi fi + + set -- } options_help='all' -options_branch='all fetch force' +options_branch='ALL all fetch force squash_branch use_tree_filter' options_clean='ALL all force' options_clone='branch edit force message method' options_config='force' options_commit='edit fetch force message' -options_fetch='all branch remote' +options_fetch='ALL all branch remote squash_branch use_tree_filter' options_init='branch remote method' -options_pull='all branch edit force message remote update' -options_push='all branch force remote squash update' +options_pull='ALL all branch edit force message remote update squash_branch use_tree_filter' +options_push='ALL all branch force remote squash update squash_branch use_tree_filter' options_status='ALL all fetch' check_option() { local var="options_${command//-/_}" - [[ ${!var} =~ $1 ]] || + [[ ${!var} =~ (^|[[:space:]])$1([[:space:]]|$) ]] || usage-error "Invalid option '--$1' for '$command'." } @@ -1142,10 +1288,11 @@ Use the --force flag to override this check or remove the worktree with fi # Set refs_ variables: - refs_subrepo_branch="refs/subrepo/$subref/branch" - refs_subrepo_commit="refs/subrepo/$subref/commit" - refs_subrepo_fetch="refs/subrepo/$subref/fetch" - refs_subrepo_push="refs/subrepo/$subref/push" + local flatSubref=$(subrepo-branch-name) + refs_subrepo_branch="refs/$flatSubref/branch" + refs_subrepo_commit="refs/$flatSubref/commit" + refs_subrepo_fetch="refs/$flatSubref/fetch" + refs_subrepo_push="refs/$flatSubref/push" # Read/parse the .gitrepo file (unless clone/init; doesn't exist yet) if [[ ! $command =~ ^(clone|init)$ ]]; then @@ -1223,7 +1370,16 @@ guess-subdir() { # encode-subdir() { subref=$subdir - if [[ ! $subref ]] || git check-ref-format "subrepo/$subref"; then + + #it will replace '/' with '-' in the subref. + local branchName=$(subrepo-branch-name) + + # + # there was an issue with subrepo/@. + # git-ref-format passed with it, but git worktree add failed. + # I added an additional check her for 'subref' itself + # + if [[ ! $subref ]] || ( git check-ref-format "$branchName" && git check-ref-format "$subref" ); then return fi @@ -1284,10 +1440,16 @@ encode-subdir() { ## 9. They cannot be the single character @. ## Note: 'subrepo/' be will prefixed, so this is always true. + ## not anylonger true. it dies with a git message in 2.22 + subref=${subref//@/%40} + ## 10. They cannot contain a \. subref=${subref//\\/%5c} + ## 11. begin with minus + subref=${subref//-/%5d} + subref=$(git check-ref-format --normalize --allow-onelevel "$subref") || error "Can't determine valid subref from '$subdir'." } @@ -1504,8 +1666,12 @@ assert-subdir-empty() { # Find all the current subrepos by looking for all the subdirectories that # contain a `.gitrepo` file. get-all-subrepos() { + local top="." + if [ "x$1" != "x" ]; then + top="$*" + fi local paths=($( - find . -name '.gitrepo' | + find $top -name '.gitrepo' | grep -v '/.git/' | grep '/.gitrepo$' | sed 's/.gitrepo$//' | @@ -1516,6 +1682,7 @@ get-all-subrepos() { for path in "${paths[@]}"; do add-subrepo "$path" done + } add-subrepo() { diff --git a/lib/git-subrepo.d/help-functions.bash b/lib/git-subrepo.d/help-functions.bash index 10cda704..4a144cd4 100644 --- a/lib/git-subrepo.d/help-functions.bash +++ b/lib/git-subrepo.d/help-functions.bash @@ -37,7 +37,9 @@ help:branch() { Use the `--force` option to write over an existing `subrepo/` branch. - The `branch` command accepts the `--all`, `--fetch` and `--force` options. + Use the `--squash_branch` option to squash all subrepo history into a single commit. + + The `branch` command accepts the `--all`, `-ALL`, `--fetch`, `--force`, and `--squash_branch` options. ... } @@ -248,6 +250,8 @@ help:pull() { specify a `--rebase`, `--merge` or `--force` strategy. The latter is the same as a `clone --force` operation, using the current remote and branch. + Use the `--squash_branch` option to squash all subrepo branch history into a single commit. + Like the `clone` command, `pull` will squash all the changes (since the last pull or clone) into one commit. This keeps your mainline history nice and clean. You can easily see the subrepo's history with the `git log` command: @@ -256,8 +260,8 @@ help:pull() { The set of commands used above are described in detail below. - The `pull` command accepts the `--all`, `--branch=`, `--edit`, `--force`, - `--message=`, `--remote=` and `--update` options. + The `pull` command accepts the `--all`, `-ALL`, `--branch=`, `--edit`, `--force`, + `--message=`, `--remote=`, `--squash_branch`, and `--update` options. ... } @@ -289,8 +293,10 @@ help:push() { discouraged. Only use this option if you fully understand it. (The `--force` option will NOT check for a proper merge. ANY branch will be force pushed!) - The `push` command accepts the `--all`, `--branch=`, `--dry-run`, `--force`, - `--merge`, `--rebase`, `--remote=`, `--squash` and `--update` options. + Use the `--squash_branch` option to squash all subrepo branch history into a single commit. + + The `push` command accepts the `--all`, `-ALL`, `--branch=`, `--dry-run`, `--force`, + `--merge`, `--rebase`, `--remote=`, `--squash`, `--squash_branch`, and `--update` options. ... } diff --git a/test/nested.t b/test/nested.t new file mode 100644 index 00000000..20110be2 --- /dev/null +++ b/test/nested.t @@ -0,0 +1,252 @@ +#!/usr/bin/env bash + +set -e + +source test/setup + +use Test::More + + +#clone-foo-and-bar + +#subrepo-clone-bar-into-foo + +curdir=$PWD + +export GIT_AUTHOR_DATE="Wed Feb 16 14:00 2037 +0100" +export GIT_AUTHOR_NAME="John Doe" +export GIT_AUTHOR_EMAIL="jain@doe.com" + +setup-nested-repo() { + workdir="$OWNER/nested" + if [ -e $workdir ]; then rm -rf $workdir; fi + + mkdir -p $workdir + cd $workdir + workdir=$PWD + + for n in {1..6}; do + cd $workdir + + mkdir -p s$n.ws + cd s$n.ws + git init + date > f$n.txt + git add f$n.txt + git commit -m "added f$n.txt to s$n.ws" + cd - + git clone -q --bare s$n.ws s$n.git + done + + cd $workdir + mkdir -p subrepos + cd subrepos + git init + date > top.txt + git add top.txt + git commit -m "added top.txt to top" + + for n in {1..3}; do + git subrepo clone ../s$n.git s$n + done + for n in {4..5}; do + git subrepo clone ../s$n.git s1/s$n + done + + git subrepo clone ../s6.git s1/s5/s6 + + echo "workdir: $PWD" +} + +setup-nested-repo + +#before="$(date -r $workdir/subrepos '+%s')" + + cd $workdir/subrepos/s2 + add-new-files s2.txt + cd $workdir/subrepos/s1 + add-new-files s1.txt + cd $workdir/subrepos/s1/s4 + add-new-files s4.txt + cd $workdir/subrepos/s1/s5/s6 + add-new-files s6.txt + + +is "$( + cd $workdir/subrepos + git subrepo branch --ALL + )"\ + "Created branch 'subrepo/s1' and worktree '.git/tmp/subrepo/s1'. +Created branch 'subrepo/s1-s4' and worktree '.git/tmp/subrepo/s1-s4'. +Created branch 'subrepo/s1-s5' and worktree '.git/tmp/subrepo/s1-s5'. +Created branch 'subrepo/s1-s5-s6' and worktree '.git/tmp/subrepo/s1-s5-s6'. +Created branch 'subrepo/s2' and worktree '.git/tmp/subrepo/s2'. +Created branch 'subrepo/s3' and worktree '.git/tmp/subrepo/s3'."\ + "branches created correctly" + +# Make sure that time stamps differ +#sleep 1 + +# is "$( +# cd $workdir/subrepos +# git push s2 +# )" \ +# "Created branch 'subrepo/bar' and worktree '.git/tmp/subrepo/bar'." \ +# "subrepo branch command output is correct" + + +#after="$(date -r $OWNER/foo/Foo '+%s')" +#assert-original-state $OWNER/foo bar + +# Check that we haven't checked out any temporary files +#is "$before" "$after" \ +# "No modification on Foo" + +test-exists "$workdir/subrepos/.git/tmp/subrepo/s1/" +test-exists "$workdir/subrepos/.git/tmp/subrepo/s2/" +test-exists "$workdir/subrepos/.git/tmp/subrepo/s3/" +test-exists "$workdir/subrepos/.git/tmp/subrepo/s1-s4/" +test-exists "!$workdir/subrepos/.git/tmp/subrepo/s1/s4/" +test-exists "$workdir/subrepos/.git/tmp/subrepo/s1-s5-s6/" + +test-exists "$workdir/subrepos/.git/refs/heads/subrepo/s1" +test-exists "$workdir/subrepos/.git/refs/heads/subrepo/s2" +test-exists "$workdir/subrepos/.git/refs/heads/subrepo/s3" +test-exists "$workdir/subrepos/.git/refs/heads/subrepo/s1-s4" +test-exists "!$workdir/subrepos/.git/refs/heads/subrepo/s1/s4" +test-exists "$workdir/subrepos/.git/refs/heads/subrepo/s1-s5-s6" + +cd $workdir/subrepos +git subrepo clean --ALL +is "$(cd $workdir/subrepos; git subrepo push --ALL)" \ + "Subrepo 's1' pushed to '../s1.git' (master). +Subrepo 's1/s4' pushed to '../s4.git' (master). +Subrepo 's1/s5' has no new commits to push. +Subrepo 's1/s5/s6' pushed to '../s6.git' (master). +Subrepo 's2' pushed to '../s2.git' (master). +Subrepo 's3' has no new commits to push." \ + "subrepo push is done correctly" + +cd $workdir/s1.ws +git pull -q ../s1.git +test-exists "s1.txt" +test-exists "f1.txt" +test-exists "!.gitrepo" +test-exists "!s3/" + +cd $workdir/s2.ws +git pull -q ../s2.git +test-exists s2.txt + +cd $workdir/s4.ws +git pull -q ../s4.git +test-exists "s4.txt" +test-exists "f4.txt" + +cd $workdir/s6.ws +git pull -q ../s6.git +test-exists "s6.txt" + + +######################### +## check branch squasning +######################### +subrepos=$workdir/subrepos + +cd $subrepos/s1/s5 +add-new-files sq5-1.txt +add-new-files sq5-2.txt +add-new-files sq5-3.txt + +cd $subrepos/s1 +add-new-files sq1-1.txt + +cd $subrepos + +### +is "$(git subrepo branch -S -F -f s1)" \ + "Created branch 'subrepo/s1' and worktree '.git/tmp/subrepo/s1'." \ + "Squashed subrepo branch s1 created" + +### +is "$(git log --format="%B" subrepo/s1 | sed 's/===.*/===/' | grep -v -P 'merged:|commit:|version:|^\s*$')" \ + 'Squashed multipe commits +=== +Author: John Doe +Email: jain@doe.com +Date: Mon Feb 16 14:00:00 2037 +0100 +git subrepo push s1 +subrepo: + subdir: "s1" +upstream: + origin: "../s1.git" + branch: "master" +git-subrepo: + origin: "https://github.com/ingydotnet/git-subrepo.git" +=== +Author: John Doe +Email: jain@doe.com +Date: Mon Feb 16 14:00:00 2037 +0100 +add new file: sq1-1.txt +add new file: s1.txt +added f1.txt to s1.ws' \ + "squashed branch s1 created correctly" + +### +is "$(git subrepo branch -S -F -f s1/s5)" \ + "Created branch 'subrepo/s1-s5' and worktree '.git/tmp/subrepo/s1-s5'." \ + "Squashed subrepo branch s1/s5 created" + +### + +is "$(git log --format="%B" subrepo/s1-s5 | sed 's/===.*/===/' | grep -v -P 'merged:|commit:|version:|^\s*$')" \ + 'Squashed multipe commits +=== +Author: John Doe +Email: jain@doe.com +Date: Mon Feb 16 14:00:00 2037 +0100 +git subrepo clone ../s5.git s1/s5 +subrepo: + subdir: "s1/s5" +upstream: + origin: "../s5.git" + branch: "master" +git-subrepo: + origin: "https://github.com/ingydotnet/git-subrepo.git" +=== +Author: John Doe +Email: jain@doe.com +Date: Mon Feb 16 14:00:00 2037 +0100 +add new file: sq5-1.txt +=== +Author: John Doe +Email: jain@doe.com +Date: Mon Feb 16 14:00:00 2037 +0100 +add new file: sq5-2.txt +=== +Author: John Doe +Email: jain@doe.com +Date: Mon Feb 16 14:00:00 2037 +0100 +add new file: sq5-3.txt +added f5.txt to s5.ws' \ + "branch s1/s5 created correctly" + + +git subrepo clean s1 +#do not clean s5 + +is "$(git subrepo push --ALL -S)" \ + "Subrepo 's1' pushed to '../s1.git' (master). +Subrepo 's1/s4' has no new commits to push. +Subrepo 's1/s5' pushed to '../s5.git' (master). +Subrepo 's1/s5/s6' has no new commits to push. +Subrepo 's2' has no new commits to push. +Subrepo 's3' has no new commits to push." \ + "push --ALL with branch squashig was done correctly" + + +done_testing + +teardown + +