From b43f82a3409183a3108a9884302a2e7c3ebd378e Mon Sep 17 00:00:00 2001 From: Laurent Bercot Date: Thu, 23 Oct 2025 01:52:03 +0000 Subject: Rename to, and improve, cgiwrapper-nollmcrawler --- .gitignore | 2 +- package/deps.mak | 5 +- package/modes | 2 +- package/targets.mak | 2 +- src/misc/cgit-nollmcrawler.c | 140 ----------------------------- src/misc/cgiwrapper-nollmcrawler.c | 143 ++++++++++++++++++++++++++++++ src/misc/deps-exe/cgit-nollmcrawler | 1 - src/misc/deps-exe/cgiwrapper-nollmcrawler | 1 + 8 files changed, 149 insertions(+), 147 deletions(-) delete mode 100644 src/misc/cgit-nollmcrawler.c create mode 100644 src/misc/cgiwrapper-nollmcrawler.c delete mode 100644 src/misc/deps-exe/cgit-nollmcrawler create mode 100644 src/misc/deps-exe/cgiwrapper-nollmcrawler diff --git a/.gitignore b/.gitignore index 589246f..0d33be1 100644 --- a/.gitignore +++ b/.gitignore @@ -11,4 +11,4 @@ /tipidee-config-preprocess /tipidee-logaggregate /ls.cgi -/cgit-nollmcrawler.cgi +/cgiwrapper-nollmcrawler diff --git a/package/deps.mak b/package/deps.mak index ad04202..5fd8da0 100644 --- a/package/deps.mak +++ b/package/deps.mak @@ -63,7 +63,6 @@ src/libtipidee/tipidee_util_defaulttext.o src/libtipidee/tipidee_util_defaulttex src/libtipidee/tipidee_util_htmlescape.o src/libtipidee/tipidee_util_htmlescape.lo: src/libtipidee/tipidee_util_htmlescape.c src/include/tipidee/util.h src/libtipidee/tipidee_util_httpdate.o src/libtipidee/tipidee_util_httpdate.lo: src/libtipidee/tipidee_util_httpdate.c src/include/tipidee/util.h src/libtipidee/tipidee_util_parse_range.o src/libtipidee/tipidee_util_parse_range.lo: src/libtipidee/tipidee_util_parse_range.c src/include/tipidee/util.h -src/misc/cgit-nollmcrawler.o src/misc/cgit-nollmcrawler.lo: src/misc/cgit-nollmcrawler.c src/misc/ls.cgi.o src/misc/ls.cgi.lo: src/misc/ls.cgi.c src/include/tipidee/response.h src/include/tipidee/util.h src/misc/tipidee-logaggregate.o src/misc/tipidee-logaggregate.lo: src/misc/tipidee-logaggregate.c src/tipideed/cgi.o src/tipideed/cgi.lo: src/tipideed/cgi.c src/include/tipidee/tipidee.h src/tipideed/tipideed-internal.h @@ -93,8 +92,8 @@ libtipidee.so.xyzzy: EXTRA_LIBS := -lskarnet libtipidee.so.xyzzy:src/libtipidee/tipidee_conf_free.lo src/libtipidee/tipidee_conf_get.lo src/libtipidee/tipidee_conf_get_argv.lo src/libtipidee/tipidee_conf_get_content_type.lo src/libtipidee/tipidee_conf_get_errorfile.lo src/libtipidee/tipidee_conf_get_redirection.lo src/libtipidee/tipidee_conf_get_resattr.lo src/libtipidee/tipidee_conf_get_resattr1.lo src/libtipidee/tipidee_conf_get_responseheaders.lo src/libtipidee/tipidee_conf_get_string.lo src/libtipidee/tipidee_conf_get_uint32.lo src/libtipidee/tipidee_conf_init.lo src/libtipidee/tipidee_headers_get_content_length.lo src/libtipidee/tipidee_headers_init.lo src/libtipidee/tipidee_headers_parse.lo src/libtipidee/tipidee_headers_search.lo src/libtipidee/tipidee_log_answer.lo src/libtipidee/tipidee_log_exit.lo src/libtipidee/tipidee_log_request.lo src/libtipidee/tipidee_log_resource.lo src/libtipidee/tipidee_log_start.lo src/libtipidee/tipidee_method.lo src/libtipidee/tipidee_response_error_nofile.lo src/libtipidee/tipidee_response_error_nofile_G.lo src/libtipidee/tipidee_response_file.lo src/libtipidee/tipidee_response_file_G.lo src/libtipidee/tipidee_response_header_date.lo src/libtipidee/tipidee_response_header_date_G.lo src/libtipidee/tipidee_response_header_date_fmt.lo src/libtipidee/tipidee_response_header_end.lo src/libtipidee/tipidee_response_header_lastmodified.lo src/libtipidee/tipidee_response_header_preparebuiltin.lo src/libtipidee/tipidee_response_header_write.lo src/libtipidee/tipidee_response_header_writeall.lo src/libtipidee/tipidee_response_header_writeall_G.lo src/libtipidee/tipidee_response_header_writemerge.lo src/libtipidee/tipidee_response_header_writemerge_G.lo src/libtipidee/tipidee_response_partial.lo src/libtipidee/tipidee_response_partial_G.lo src/libtipidee/tipidee_response_status.lo src/libtipidee/tipidee_rql_read.lo src/libtipidee/tipidee_uri_parse.lo src/libtipidee/tipidee_util_chunked_read.lo src/libtipidee/tipidee_util_defaulttext.lo src/libtipidee/tipidee_util_htmlescape.lo src/libtipidee/tipidee_util_httpdate.lo src/libtipidee/tipidee_util_parse_range.lo libtipidee.dylib.xyzzy: EXTRA_LIBS := -lskarnet libtipidee.dylib.xyzzy:src/libtipidee/tipidee_conf_free.lo src/libtipidee/tipidee_conf_get.lo src/libtipidee/tipidee_conf_get_argv.lo src/libtipidee/tipidee_conf_get_content_type.lo src/libtipidee/tipidee_conf_get_errorfile.lo src/libtipidee/tipidee_conf_get_redirection.lo src/libtipidee/tipidee_conf_get_resattr.lo src/libtipidee/tipidee_conf_get_resattr1.lo src/libtipidee/tipidee_conf_get_responseheaders.lo src/libtipidee/tipidee_conf_get_string.lo src/libtipidee/tipidee_conf_get_uint32.lo src/libtipidee/tipidee_conf_init.lo src/libtipidee/tipidee_headers_get_content_length.lo src/libtipidee/tipidee_headers_init.lo src/libtipidee/tipidee_headers_parse.lo src/libtipidee/tipidee_headers_search.lo src/libtipidee/tipidee_log_answer.lo src/libtipidee/tipidee_log_exit.lo src/libtipidee/tipidee_log_request.lo src/libtipidee/tipidee_log_resource.lo src/libtipidee/tipidee_log_start.lo src/libtipidee/tipidee_method.lo src/libtipidee/tipidee_response_error_nofile.lo src/libtipidee/tipidee_response_error_nofile_G.lo src/libtipidee/tipidee_response_file.lo src/libtipidee/tipidee_response_file_G.lo src/libtipidee/tipidee_response_header_date.lo src/libtipidee/tipidee_response_header_date_G.lo src/libtipidee/tipidee_response_header_date_fmt.lo src/libtipidee/tipidee_response_header_end.lo src/libtipidee/tipidee_response_header_lastmodified.lo src/libtipidee/tipidee_response_header_preparebuiltin.lo src/libtipidee/tipidee_response_header_write.lo src/libtipidee/tipidee_response_header_writeall.lo src/libtipidee/tipidee_response_header_writeall_G.lo src/libtipidee/tipidee_response_header_writemerge.lo src/libtipidee/tipidee_response_header_writemerge_G.lo src/libtipidee/tipidee_response_partial.lo src/libtipidee/tipidee_response_partial_G.lo src/libtipidee/tipidee_response_status.lo src/libtipidee/tipidee_rql_read.lo src/libtipidee/tipidee_uri_parse.lo src/libtipidee/tipidee_util_chunked_read.lo src/libtipidee/tipidee_util_defaulttext.lo src/libtipidee/tipidee_util_htmlescape.lo src/libtipidee/tipidee_util_httpdate.lo src/libtipidee/tipidee_util_parse_range.lo -cgit-nollmcrawler: EXTRA_LIBS := -cgit-nollmcrawler: src/misc/cgit-nollmcrawler.o -lskarnet +cgiwrapper-nollmcrawler: EXTRA_LIBS := +cgiwrapper-nollmcrawler: src/misc/cgiwrapper-nollmcrawler.o -lskarnet ls.cgi: EXTRA_LIBS := ls.cgi: src/misc/ls.cgi.o libtipidee.a.xyzzy -lskarnet tipidee-logaggregate: EXTRA_LIBS := diff --git a/package/modes b/package/modes index 9602f3f..76f0a89 100644 --- a/package/modes +++ b/package/modes @@ -3,4 +3,4 @@ tipidee-config-preprocess 0755 tipidee-config 0755 ls.cgi 0755 tipidee-logaggregate 0755 -cgit-nollmcrawler 0755 +cgiwrapper-nollmcrawler 0755 diff --git a/package/targets.mak b/package/targets.mak index 87d871a..d5bd14d 100644 --- a/package/targets.mak +++ b/package/targets.mak @@ -2,7 +2,7 @@ BIN_TARGETS := \ tipideed \ tipidee-config \ tipidee-logaggregate \ -cgit-nollmcrawler +cgiwrapper-nollmcrawler LIBEXEC_TARGETS := \ tipidee-config-preprocess \ diff --git a/src/misc/cgit-nollmcrawler.c b/src/misc/cgit-nollmcrawler.c deleted file mode 100644 index 71d31ae..0000000 --- a/src/misc/cgit-nollmcrawler.c +++ /dev/null @@ -1,140 +0,0 @@ -/* ISC license. */ - -#include -#include -#include -#include -#include - -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#define USAGE "as a CGI script: cgit-nollmcrawler [ -v verbosity ] [ -d pathinfodepth ] rulesdir badregex realcgit..." -#define dieusage() strerr_dieusage(100, USAGE) - -enum golb_e -{ - GOLB_FORCE = 0x01 -} ; - -static gol_bool const rgolb[] = -{ - { .so = 'f', .lo = "force", .clear = 0, .set = GOLB_FORCE } -} ; - -enum gola_e -{ - GOLA_VERBOSITY, - GOLA_DEPTH, - GOLA_N -} ; - -static gol_arg const rgola[] = -{ - { .so = 'v', .lo = "verbosity", .i = GOLA_VERBOSITY }, - { .so = 'd', .lo = "pathinfo-depth", .i = GOLA_DEPTH } -} ; - -int main (int argc, char const *const *argv) -{ - unsigned int verbosity = 1 ; - char const *wgola[GOLA_N] = { 0 } ; - uint64_t wgolb = 0 ; - unsigned int golc ; - unsigned int depth = 1 ; - char const *remoteaddr ; - char const *x ; - size_t rdlen, m = 0 ; - char ip[16] ; - int is6 ; - PROG = "cgit-nollmcrawler" ; - - golc = GOL_main(argc, argv, rgolb, rgola, &wgolb, wgola) ; - argc -= golc ; argv += golc ; - if (wgola[GOLA_VERBOSITY] && !uint0_scan(wgola[GOLA_VERBOSITY], &verbosity)) - strerr_dief1x(100, "verbosity needs to be an unsigned integer") ; - if (wgola[GOLA_DEPTH] && !uint0_scan(wgola[GOLA_DEPTH], &depth)) - strerr_dief1x(100, "pathinfo-depth needs to be an unsigned integer") ; - if (argc < 3) dieusage() ; - - remoteaddr = getenv("REMOTE_ADDR") ; - if (ip6_scan(remoteaddr, ip)) is6 = 1 ; - else if (ip4_scan(remoteaddr, ip)) is6 = 0 ; - else strerr_dieinvalid(100, "REMOTE_ADDR") ; - - rdlen = strlen(argv[0]) ; - char fn[rdlen + IP6_FMT + 16] ; - - memcpy(fn + m, argv[0], rdlen) ; m += rdlen ; - memcpy(fn + m, "/ip", 3) ; m += 3 ; - fn[m++] = is6 ? '6' : '4' ; - fn[m++] = '/' ; - m += is6 ? ip6_fmt(fn + m, ip) : ip4_fmt(fn + m, ip) ; - fn[m++] = '_' ; - memcpy(fn + m, is6 ? "128" : "32", is6 ? 3 : 2) ; m += 2 + is6 ; - fn[m] = 0 ; - memcpy(fn + m, "/allow", 7) ; - if (access(fn, W_OK) == 0) goto allow ; - if (errno != ENOENT) strerr_diefu2sys(111, "access ", fn) ; - memcpy(fn + m + 1, "deny", 5) ; - if (access(fn, W_OK) == 0) goto deny ; - if (errno != ENOENT) strerr_diefu2sys(111, "access ", fn) ; - fn[m] = 0 ; - - x = getenv("PATH_INFO") ; - if (!x) goto writeandallow ; - if (byte_count(x, strlen(x), '/') <= depth) goto writeandallow ; - x = getenv("QUERY_STRING") ; - if (!x) goto writeandallow ; - - { - regex_t re ; - int e = regcomp(&re, argv[1], REG_EXTENDED) ; - if (e == REG_BADPAT) strerr_dief2x(100, "invalid regex: ", argv[1]) ; - if (e) - { - char fmt[INT_FMT] ; - fmt[int_fmt(fmt, e)] = 0 ; - strerr_diefu4x(111, "regcomp ", argv[1], ": error code is ", fmt) ; - } - e = regexec(&re, x, 0, 0, REG_NOSUB) ; - if (e == 0) goto writeanddeny ; - if (e != REG_NOMATCH) - { - char fmt[INT_FMT] ; - fmt[int_fmt(fmt, e)] = 0 ; - strerr_diefu4x(111, "regexec ", argv[1], ": error code is ", fmt) ; - } - // regfree(&re) ; - } - - writeandallow: - if (symlink("../outputs/allow", fn) == -1) - { - if (errno != EEXIST) strerr_diefu4sys(111, "symlink ", "../outputs/allow", " to ", fn) ; - } - if (verbosity >= 2) - { - x = getenv("REQUEST_URI") ; - strerr_warni("allowing ", remoteaddr, " requesting ", x) ; - } - allow: - exec(argv + 2) ; - strerr_dieexec(111, argv[2]) ; - - writeanddeny: - if (symlink("../outputs/deny", fn) == -1) - { - if (errno != EEXIST) strerr_diefu4sys(111, "symlink ", "../outputs/deny", " to ", fn) ; - } - deny: - buffer_putsflush(buffer_1small, "Status: 403 Go fuck yourself, crawler\nContent-Length: 0\n\n") ; - _exit(0) ; -} diff --git a/src/misc/cgiwrapper-nollmcrawler.c b/src/misc/cgiwrapper-nollmcrawler.c new file mode 100644 index 0000000..95a2ea5 --- /dev/null +++ b/src/misc/cgiwrapper-nollmcrawler.c @@ -0,0 +1,143 @@ +/* ISC license. */ + +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#define USAGE "as a CGI script: cgiwrapper-nollmcrawler [ -v verbosity ] [ -d pathinfodepth ] rulesdir badregex realcgit..." +#define dieusage() strerr_dieusage(100, USAGE) + +enum golb_e +{ + GOLB_FORCE = 0x01 +} ; + +static gol_bool const rgolb[] = +{ + { .so = 'f', .lo = "force", .clear = 0, .set = GOLB_FORCE } +} ; + +enum gola_e +{ + GOLA_VERBOSITY, + GOLA_DEPTH, + GOLA_N +} ; + +static gol_arg const rgola[] = +{ + { .so = 'v', .lo = "verbosity", .i = GOLA_VERBOSITY }, + { .so = 'd', .lo = "pathinfo-depth", .i = GOLA_DEPTH } +} ; + +int main (int argc, char const *const *argv) +{ + unsigned int verbosity = 1 ; + char const *wgola[GOLA_N] = { 0 } ; + uint64_t wgolb = 0 ; + unsigned int golc ; + unsigned int depth = 0 ; + char const *remoteaddr ; + char const *x ; + size_t rdlen, m = 0 ; + char ip[16] ; + int is6 ; + PROG = "cgit-nollmcrawler" ; + + golc = GOL_main(argc, argv, rgolb, rgola, &wgolb, wgola) ; + argc -= golc ; argv += golc ; + if (wgola[GOLA_VERBOSITY] && !uint0_scan(wgola[GOLA_VERBOSITY], &verbosity)) + strerr_dief1x(100, "verbosity needs to be an unsigned integer") ; + if (wgola[GOLA_DEPTH] && !uint0_scan(wgola[GOLA_DEPTH], &depth)) + strerr_dief1x(100, "pathinfo-depth needs to be an unsigned integer") ; + if (argc < 3) dieusage() ; + + remoteaddr = getenv("REMOTE_ADDR") ; + if (ip6_scan(remoteaddr, ip)) is6 = 1 ; + else if (ip4_scan(remoteaddr, ip)) is6 = 0 ; + else strerr_dieinvalid(100, "REMOTE_ADDR") ; + + rdlen = strlen(argv[0]) ; + char fn[rdlen + IP6_FMT + 16] ; + + memcpy(fn + m, argv[0], rdlen) ; m += rdlen ; + memcpy(fn + m, "/ip", 3) ; m += 3 ; + fn[m++] = is6 ? '6' : '4' ; + fn[m++] = '/' ; + m += is6 ? ip6_fmt(fn + m, ip) : ip4_fmt(fn + m, ip) ; + fn[m++] = '_' ; + memcpy(fn + m, is6 ? "128" : "32", is6 ? 3 : 2) ; m += 2 + is6 ; + fn[m] = 0 ; + memcpy(fn + m, "/allow", 7) ; + if (access(fn, W_OK) == 0) goto allow ; + if (errno != ENOENT) strerr_diefu2sys(111, "access ", fn) ; + memcpy(fn + m + 1, "deny", 5) ; + if (access(fn, W_OK) == 0) goto deny ; + if (errno != ENOENT) strerr_diefu2sys(111, "access ", fn) ; + fn[m] = 0 ; + + if (depth) + { + x = getenv("PATH_INFO") ; + if (!x) goto writeandallow ; + if (byte_count(x, strlen(x), '/') <= depth) goto writeandallow ; + } + x = getenv("QUERY_STRING") ; + if (!x) goto writeandallow ; + + { + regex_t re ; + int e = regcomp(&re, argv[1], REG_EXTENDED) ; + if (e == REG_BADPAT) strerr_dief2x(100, "invalid regex: ", argv[1]) ; + if (e) + { + char fmt[INT_FMT] ; + fmt[int_fmt(fmt, e)] = 0 ; + strerr_diefu4x(111, "regcomp ", argv[1], ": error code is ", fmt) ; + } + e = regexec(&re, x, 0, 0, REG_NOSUB) ; + if (e == 0) goto writeanddeny ; + if (e != REG_NOMATCH) + { + char fmt[INT_FMT] ; + fmt[int_fmt(fmt, e)] = 0 ; + strerr_diefu4x(111, "regexec ", argv[1], ": error code is ", fmt) ; + } + // regfree(&re) ; + } + + writeandallow: + if (symlink("../outputs/allow", fn) == -1) + { + if (errno != EEXIST) strerr_diefu4sys(111, "symlink ", "../outputs/allow", " to ", fn) ; + } + if (verbosity >= 2) + { + x = getenv("REQUEST_URI") ; + strerr_warni("allowing ", remoteaddr, " requesting ", x) ; + } + allow: + exec(argv + 2) ; + strerr_dieexec(111, argv[2]) ; + + writeanddeny: + if (symlink("../outputs/deny", fn) == -1) + { + if (errno != EEXIST) strerr_diefu4sys(111, "symlink ", "../outputs/deny", " to ", fn) ; + } + deny: + buffer_putsflush(buffer_1small, "Status: 403 Go fuck yourself, crawler\nContent-Length: 0\n\n") ; + _exit(0) ; +} diff --git a/src/misc/deps-exe/cgit-nollmcrawler b/src/misc/deps-exe/cgit-nollmcrawler deleted file mode 100644 index e7187fe..0000000 --- a/src/misc/deps-exe/cgit-nollmcrawler +++ /dev/null @@ -1 +0,0 @@ --lskarnet diff --git a/src/misc/deps-exe/cgiwrapper-nollmcrawler b/src/misc/deps-exe/cgiwrapper-nollmcrawler new file mode 100644 index 0000000..e7187fe --- /dev/null +++ b/src/misc/deps-exe/cgiwrapper-nollmcrawler @@ -0,0 +1 @@ +-lskarnet -- cgit v1.3.1