Stdin, Files, and Pipes

Stdin, Files, and Pipes

Overview

Great CLIs work in pipelines: read files or stdin, write transform results to stdout. This chapter is pure stdlib patterns for input discovery, streaming, and safe file handling.

Input priority (common UX)

1. Explicit file args
2. If no args (or arg is "-") → stdin

func openInputs(args []string) ([]io.ReadCloser, error) {
    if len(args) == 0 {
        return []io.ReadCloser{io.NopCloser(os.Stdin)}, nil
    }
    var out []io.ReadCloser
    for _, a := range args {
        if a == "-" {
            out = append(out, io.NopCloser(os.Stdin))
            continue
        }
        f, err := os.Open(a)
        if err != nil {
            // close already opened on error
            for _, c := range out {
                c.Close()
            }
            return nil, err
        }
        out = append(out, f)
    }
    return out, nil
}

Stream line by line

func process(r io.Reader, w io.Writer, pred func(string) bool) error {
    sc := bufio.NewScanner(r)
    // optional: larger tokens
    // buf := make([]byte, 0, 64*1024)
    // sc.Buffer(buf, 1024*1024)
    for sc.Scan() {
        line := sc.Text()
        if pred(line) {
            if _, err := fmt.Fprintln(w, line); err != nil {
                return err
            }
        }
    }
    return sc.Err()
}

Example: grep1 (stdlib)

func main() {
    n := flag.Bool("n", false, "print line numbers")
    flag.Parse()
    if flag.NArg() < 1 {
        fmt.Fprintln(os.Stderr, "usage: grep1 [-n] <pattern> [files...]")
        os.Exit(2)
    }
    pat := flag.Arg(0)
    files := flag.Args()[1:]
    inputs, err := openInputs(files)
    if err != nil {
        fmt.Fprintln(os.Stderr, err)
        os.Exit(1)
    }
    matched := false
    lineNo := 0
    for _, in := range inputs {
        sc := bufio.NewScanner(in)
        for sc.Scan() {
            lineNo++
            line := sc.Text()
            if strings.Contains(line, pat) {
                matched = true
                if *n {
                    fmt.Printf("%d:%s\n", lineNo, line)
                } else {
                    fmt.Println(line)
                }
            }
        }
        err = sc.Err()
        in.Close()
        if err != nil {
            fmt.Fprintln(os.Stderr, err)
            os.Exit(1)
        }
    }
    if !matched {
        os.Exit(1)
    }
}

echo -e "go\nrust\ngo lang" | go run . go
printf 'a\nb\n' > t.txt && go run . -n b t.txt

Example: wc1 lines/words/bytes

func count(r io.Reader) (lines, words, bytes int, err error) {
    br := bufio.NewReader(r)
    for {
        b, e := br.ReadByte()
        if e != nil {
            if e == io.EOF {
                return lines, words, bytes, nil
            }
            return 0, 0, 0, e
        }
        bytes++
        if b == '\n' {
            lines++
        }
    }
}
// words: use bufio.Scanner with Split(bufio.ScanWords) in a second pass
// or track isSpace transitions in the byte loop

Example: JSON filter from stdin

func run(r io.Reader, w io.Writer, field string) error {
    dec := json.NewDecoder(r)
    enc := json.NewEncoder(w)
    for {
        var m map[string]any
        if err := dec.Decode(&m); err != nil {
            if err == io.EOF {
                return nil
            }
            return err
        }
        if v, ok := m[field]; ok {
            if err := enc.Encode(v); err != nil {
                return err
            }
        }
    }
}

echo '{"name":"a"}{"name":"b"}' | go run . -field name

Detect pipe vs TTY for stdin

func stdinIsPipe() bool {
    fi, err := os.Stdin.Stat()
    if err != nil {
        return false
    }
    return fi.Mode()&os.ModeCharDevice == 0
}

If stdin is a TTY and no files given, you may print usage instead of blocking forever:

if len(args) == 0 && !stdinIsPipe() {
    return fmt.Errorf("usage: tool <files> or pipe stdin")
}

(Some tools intentionally wait for interactive stdin—document either way.)

Atomic write of output files

func writeFileAtomic(path string, data []byte, mode os.FileMode) error {
    dir := filepath.Dir(path)
    tmp, err := os.CreateTemp(dir, ".tmp-*")
    if err != nil {
        return err
    }
    tmpName := tmp.Name()
    defer func() { _ = os.Remove(tmpName) }() // if not renamed

    if _, err := tmp.Write(data); err != nil {
        tmp.Close()
        return err
    }
    if err := tmp.Chmod(mode); err != nil {
        tmp.Close()
        return err
    }
    if err := tmp.Close(); err != nil {
        return err
    }
    return os.Rename(tmpName, path)
}

Progress on stderr only

fmt.Fprintf(os.Stderr, "\rprocessed %d", n)
// final newline
fmt.Fprintln(os.Stderr)

Never put \r progress on stdout if users pipe to files.

Rules of thumb

Do	Don’t
Stream with bufio	ReadAll multi-GB stdin
Accept - as stdin	Force temp files for every pipeline
Close files	Leak FDs in long loops
Errors → stderr	Corrupt the data stream

Try next

1. Implement uniq1 adjacent-line dedupe from stdin.
2. Add cut1 -f 2 -d , for CSV-ish lines.
3. Pipe grep1 into wc -l and confirm exit codes compose.