Bulk editing files on linux via the commandline

Jul 23, 2022 3 minutes to read Wolf Noble Tech Bash Ed Howto Linux

What are we trying to do?

I realized the metadata of the various files here was a little rough around the edges. That inconsistency was making it hard to have automation behave properly.

This meant I have hundreds of files which each needed just a little something different done to them.

for example, we have a list of a few files here:

1
2
3
4
docs/WPL/servers.md
docs/Observium/data.md
docs/Smallstep/setup.md
docs/Raspbian/PXE/make-it-go.md

in each of those files, we want to inject some data into the hugoFrontmatter of the page:

kind: 'page'

We want to do this Anywhere after the first line….

The first few lines of one of the files ...

1
2
3
4
5
6
7
~/Git_Repos/Wolfspyre/docs/content (wpl_main)$ head -6 Snippits/Geekdocs/Shortcodes/toc.md
---
categories: ['Hugo',Tech]
title: ToC
geekdocDescription: 'Create a Table of Content'
type: 'docs'
geekdocNav: true

But before the second set of --- characters, which signifies the end of Frontmatter to Hugo

What tools we have handy

Caveats…. There’s always some bullshit isn’t there?

Because we want to add a line that has a space, a colon, and single quotes, things get a LITTLE wonky.

not a big deal, but it does make it slightly more awkward toolchain wise…. and that makes it harder to avoid making a mistake.

so… back to The Unix Philosophy of stringing a bunch of single purpose tools together to make it go it is!

Lets do it!

Using ed

the old and trusty ed really comes to the rescue in times like this.

Contents of ed.script ...

ed.script

1
2
3
4
5
6
2i
kind: 'page'
.
w
q

so… cool…

We have a list of files which need the alteration: foo.txt.
We jave a script to drive ed

Now we need to coalesce some tooling to do the needful.

Contents of edscript.sh ...

  1
  2
  3
  4
  5
  6
  7
  8
  9
 10
 11
 12
 13
 14
 15
 16
 17
 18
 19
 20
 21
 22
 23
 24
 25
 26
 27
 28
 29
 30
 31
 32
 33
 34
 35
 36
 37
 38
 39
 40
 41
 42
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
#!/usr/bin/env bash
export MIN_OPTS=2
export DATESTRING=$(date +%m%d%y)
export _STARTTIME=$(/bin/date +%s)
usage () {
  cat<WolfspyreLabs scriptypoo:
 $0 Expects at least $(greenme ${MIN_OPTS} ) options.
    usage: $0 $(yellome "LISTFILE  EDSCRIPT")
$(yellome listfile) should be a local file that contains a list of files to be modified 
$(yellome "edscript") should be a local file. containing the scripted operations you wish performed by $(greenme "ed")  EXAMPLE:
$(redme "
whateverworkdir: ~/ cat list-of-files-to-edit.txt  $(redme '||' ) $(redme "whateverworkdir: ~/ cat ed.script") ")
$(greenme "somedirectory/file.md                              ")$(redme '||' ) $(yellome "2i")
$(greenme "dir/file01.md                                      ")$(redme '||' ) $(yellome "kind: 'page'")
$(greenme "draft/file.md                                      ")$(redme '||' ) $(yellome ".")
$(greenme "project-epic-idea/no-more-disco.md                 ")$(redme '||' ) $(yellome "w")
$(greenme "resume/egoic-prelude.md                            ")$(redme '||' ) $(yellome "q")
$(redme "whateverworkdi: ~/                                 || whateverworkdir: ~/ ")
...Make sense?
Debug statements will be output if the variable DEBUG exists; Example:
                   DEBUG=true; $0 $(yellome "LISTFILE                  EDSCRIPT")
$(yellome "whateverworkdir: ~/" )$(greenme "DEBUG=true; ${0} list-of-files.to.edit.txt ed.script")

EOF

}

runningTime () {
  _NOW=$(/bin/date +%s)
  echo -e "$(( ${_NOW}-${_STARTTIME}  ))"
}

convertsecs() {
 #take an int which is a number of seconds (IE:
 # now=$(date %s) ; sleep 30; newer_now=$(date %s); age=(${newer_now)-${now}) ; info "This took: $(convertsecs $age)"
 # ^^^^^---- untested example
 ((h=${1}/3600))
 ((m=(${1}%3600)/60))
 ((s=${1}%60))
 printf "%02d:%02d:%02d\n" ${h} ${m} ${s}
}
redme(){
  _RED=$1
  echo -e "\033[31m${_RED}\033[0m"
}
yellome(){
  _YELO=$1
  echo -e "\033[1;33m${_YELO}\033[0m"
}
greenme(){
 _GREN=$1
  echo -e "\033[32m${_GREN}\033[0m"

}
error () {
  MSG=$1
  _time=$(convertsecs $(runningTime))
  echo -e "\033[31m[${_time}] ERROR\033[0m: $MSG" >>/dev/stderr
}

warn () {
  MSG=$1
  _time=$(convertsecs $(runningTime))
  echo -e "\033[1;33m[${_time}] WARN\033[0m: $MSG" >>/dev/stderr
}

info () {
  MSG=$1
  _time=$(convertsecs $(runningTime))
  echo -e "\033[32m[${_time}] INFO\033[0m: $MSG"
}

debug () {
  MSG=$1
  _time=$(convertsecs $(runningTime))
  if [[ $2 && $2 == 'verbose' ]]; then
    verbose_msg=true
  else
    verbose_msg=false
  fi
  if [ -n  "$DEBUG" ]; then
    if $verbose_msg; then
      #only display verbose if DEBUG == verbose
      if [ ${DEBUG} == 'verbose' ]; then
        echo -e "\033[35mVERBOSE\033[0m: \033[32m[${_time} s] ${FUNCNAME[1]}()\033[0m: ${MSG}"
      fi
    else
      echo -e "\033[34mDEBUG\033[0m: \033[32m[${_time} s] ${FUNCNAME[1]}()\033[0m: ${MSG}"
    fi
  fi
}

echodo() {
  echo "${@}"
  (${@})
}

infodo() {
  info "${@}"
  (${@})
}


if [[ $# -lt ${MIN_OPTS} ]]; then
  usage

elif [[ ! -f ${1} ]]; then
  error " ::: ${1} is not a file. Cannot continue. "
  usage
  exit 1

else


_WC='/usr/bin/wc';
_AWK='/usr/bin/awk';
_CAT='/bin/cat';
_ED='/bin/ed';
_GREP='/usr/bin/grep';
_EDSCRIPT='/tmp/fix.ed';
_TESTSTRING="kind: 'page'";
_SKIP=0;
_SUCCESS=0;
_ERR=0;
_TOTAL=0;
_COUNT=0;
_R=0;
if [[ ! -f ${_EDSCRIPT} ]]; then
  error "Cannot find ed script: ${_EDSCRIPT}. nothing to do."
  exit -1
fi
_T=`${_WC} -l ${1} | ${_AWK} '{print $1}'`;
_TOTAL=$(greenme "${_T}")

for _FILENAME in `cat ${1}`; do
  _R=0;
  let _COUNT=(${_COUNT}+1);
  if [[ ! -f ${_FILENAME} ]]; then
    let _ERR=(${_ERR}+1);
    warn "[${_COUNT}/${_TOTAL}] ($(yellome "${_SKIP}")|$(greenme "${_SUCCESS}")|$(redme "${_ERR}")) [$(greenme "${_FILENAME}")]  doesn't exist. Skipping."
  else
    debug "${_FILENAME} exists"
    #check to see if the change is necessary
    if [[ `grep -q "${_TESTSTRING}" ${_FILENAME}; echo $?` -eq 0 ]]; then
      let _SKIP=(${_SKIP}+1)
      warn "[${_COUNT}/${_TOTAL}] ($(yellome "${_SKIP}")|$(greenme "${_SUCCESS}")|$(redme "${_ERR}")) [$(greenme "${_FILENAME}")] already contains $(greenme "${_TESTSTRING}"). Skipping. Skipped"
    else
      debug "[${_COUNT}/${_TOTAL}] ($(yellome "${_SKIP}")|$(greenme "${_SUCCESS}")|$(redme "${_ERR}")) [$(greenme "${_FILENAME}")] does not contain TESTSTRING: ${_TESTSTRING}. Proceeding with edit"
      # count and store the number of lines in the file currently.
      _PRECT=`${_WC} -l ${_FILENAME} | ${_AWK} '{print $1}'`
      #edit the file!
      ${_ED} ${_FILENAME} < ${_EDSCRIPT}
      _POSTCT=`${_WC} -l ${_FILENAME} | ${_AWK} '{print $1}'`
      debug "[${_COUNT}/${_TOTAL}] ($(yellome "${_SKIP}")|$(greenme "${_SUCCESS}")|$(redme "${_ERR}")) [$(greenme "${_FILENAME}")] Pre/Post Linecount: ${_PRECT}/${_POSTCT}"
      let _R=(${_POSTCT}-${_PRECT});
      if [[ ${_R} == 1 ]]; then
         let _SUCCESS=(${_SUCCESS}+1);
         info "[${_COUNT}/${_TOTAL}] ($(yellome "${_SKIP}")|$(greenme "${_SUCCESS}")|$(redme "${_ERR}")) [$(greenme "${_FILENAME}")] Success!!!  Pre/Post Linecount: ${_PRECT}/${_POSTCT} "
      elif [[ ${_R} == 2 ]]; then
         let _SUCCESS=(${_SUCCESS}+1);
         info "[${_COUNT}/${_TOTAL}] ($(yellome "${_SKIP}")|$(greenme "${_SUCCESS}")|$(redme "${_ERR}")) [$(greenme "${_FILENAME}")] Success!!!  Pre/Post Linecount: ${_PRECT}/${_POSTCT} [ newline added ]"      
      else
         let _ERR=(${_ERR}+1);
         echo ${_FILENAME} >>errors.txt
         warn "[${_COUNT}/${_TOTAL}] ($(yellome "${_SKIP}")|$(greenme "${_SUCCESS}")|$(redme "${_ERR}")) [$(greenme "${_FILENAME}")]  Post-Pre Linecount SHOULD eq 1. but I'm in an error state???: ${_PRECT}/${_POSTCT}"
      fi
    fi
  fi
done
fi