diff --git a/.gitmodules b/.gitmodules new file mode 100644 index 000000000..8bb3e999e --- /dev/null +++ b/.gitmodules @@ -0,0 +1,3 @@ +[submodule "default-assets"] + path = default-assets + url = https://github.com/GrangerHub/tremulous-default-assets.git diff --git a/.travis.yml b/.travis.yml index b420d8bb0..f497f3008 100644 --- a/.travis.yml +++ b/.travis.yml @@ -19,14 +19,14 @@ matrix: env: PLATFORM=linux before_install: - docker pull grangerhub/tremulous13:latest - - docker run -e PLATFORM -v $(pwd):/usr/src grangerhub/tremulous13:latest ./misc/docker-build.sh + - docker run -e PLATFORM -v $(pwd):/usr/src grangerhub/tremulous13:latest ./misc/travis-ci-docker-build.sh script: - true - os: linux env: PLATFORM=mingw32 before_install: - docker pull grangerhub/tremulous13:latest - - docker run -e PLATFORM -v $(pwd):/usr/src grangerhub/tremulous13:latest ./misc/docker-build.sh + - docker run -e PLATFORM -v $(pwd):/usr/src grangerhub/tremulous13:latest ./misc/travis-ci-docker-build.sh script: - true - os: linux @@ -34,7 +34,7 @@ matrix: before_install: - docker pull grangerhub/tremulous13:latest - docker run -e ARCH -e PLATFORM -v $(pwd):/usr/src grangerhub/tremulous13:latest - ./misc/docker-build.sh + ./misc/travis-ci-docker-build.sh script: - true - os: osx @@ -49,9 +49,9 @@ deploy: provider: releases skip_cleanup: true api_key: - secure: "jtNUkZL1TK3nD1LfWi5kypaL2WzUHxM/N72AcwKTmvieQ/3iwL8gRDFOhfWKFkNcyuCsEBw+c2KlpgfNGVNZYJgZrQ3Ozvnno5yN+V8Pb4ODEFU0Ps4EHSY7gsdF6bBdp3ooqsCsg8BkGfIggFsV+R9oKOyWWH1/N1jPWhf8ioWD+mMaAh5ZOsyQIECtSKAKMu9/hikVkdDuNsIAiaiJhYvP8rli5fYrZ4sVJih3YgeJj42i2iLeeWXmOmYNF7T5ywuBpaUjQJBEo83MilglTF0UOb471ADU2Gz3SMrnNCOdqDZLQaBL+QVrrgJUvXy5A92nxkY16mJaIvdDEOeIzWwgWVLoKyil6AFW5bR7lp2YW+/4I6zYEkGrpUaiZ+S1lgvm3CwVgwhg3rVuCi0qrI/7j/ImS9ORmz9DGbVgtg+y7mZnUIcEWX4jRDbbYWYIc/1fN+fGt73+41h0HS5of7d31yH6AyxB8/DlkuoKZIVHiErD0J3ZUkdVJOwhPFiPKViYR/07KEvafhkZkcffZh7arIesFB++4Kmglapb/awHJRh4lhfKd16IiCGjjfof8MDKSkmqAdXUsdCRcPu1Q0/cJaO3TRiTS6M0skrBUeIOOYsV0AdWM1Fy7dTySEEYlqZXnkiD98MG7CwobkWXmeVZHEzOJh2fDtv2w2ojoDI=" + secure: g9HIz82c3vz3bl7vsMuODCz6kwf4IogJGw8V1Pn7TapXTeO9JnR1BqAhQbuqjPEBsgnIf+zwNblzBYmSyihB5I8JC8GiyhX/5tq5C/fd0z+uAH3Gsm5BFowRRUjVQlneH61NY4LP8H6h1KfkFBlD3NfW1D8kCa7vUyEh4Lj11wJMj9WgYctYfPu0rW2f8mCBSJ5VPR0EDNnJ8DV0EKEKmv7tiF6sY6eLuMg7VMtSPCCd/NkSQz1A04xYi6AwghrCJ8EZvkBhRIqgUGFL8y2si2Dt30fEG4nz4eDuqm17JoI9NX6Iz7v8iwU2CBL4wrDlqVYcncYxFqL89ZcXAmVHBl9eqqPKfA0KzR10IbkjVuVGvM0eQTOlGayDNalutZ0UtDuqPnMjHALpH+AsGUCwdYCQN5KL6b/xsgmQYUHuRE2ThbVk6MC7UbVSj6BQPiff9q41lSlHin2tkL9XUrGea0RY9d8F5dgT1oiYygUOpXcXA8slme2zvkuaNDrgFgkqU9ii3zqhqJxcFHkDIOmqxJJhyKcI4v6BWCz0evZzV4Qv3LpQyZeKJhmCAH5yU3xJYwV5ePoHrEeZc4a9y46V6R2hMttwawyR1TPZeT4PvEAFtusmfpmVeVlLhB9mmINK/D9LmuTEo0CdBioCgsIM1YQLFT8727yC/acB61dYowE= file_glob: true file: build/*.zip on: - repo: GrangerHub/tremulous + repo: dGr8LookinSparky/tremulous tags: true diff --git a/Makefile b/Makefile index ba7f9acb6..826110f3e 100644 --- a/Makefile +++ b/Makefile @@ -42,6 +42,9 @@ endif ifndef BUILD_RENDERER_OPENGL2 BUILD_RENDERER_OPENGL2= endif +ifndef BUILD_DATA_PK3 + BUILD_DATA_PK3= +endif ############################################################################# # @@ -120,11 +123,15 @@ SERVERBIN=tremded endif ifndef BASEGAME -BASEGAME=gpp +BASEGAME=trem13 endif BASEGAME_CFLAGS=-I../../${MOUNT_DIR} +ifndef BASEMOD +BASEMOD=basemod +endif + ifndef COPYDIR COPYDIR="/usr/local/games/tremulous" endif @@ -253,8 +260,49 @@ endif ############################################################################# -BD=$(BUILD_DIR)/debug-$(PLATFORM)-$(ARCH) -BR=$(BUILD_DIR)/release-$(PLATFORM)-$(ARCH) +ifeq ($(ARCH),x86_64) +ifeq ($(PLATFORM),mingw32) + BD=$(BUILD_DIR)/debug-windows-64 + BR=$(BUILD_DIR)/release-windows-64 +else +ifeq ($(PLATFORM),darwin) + BD=$(BUILD_DIR)/debug-macos-64 + BR=$(BUILD_DIR)/release-macos-64 +else + BD=$(BUILD_DIR)/debug-$(PLATFORM)-64 + BR=$(BUILD_DIR)/release-$(PLATFORM)-64 +endif +endif +else +ifeq ($(ARCH),x86) +ifeq ($(PLATFORM),mingw32) + BD=$(BUILD_DIR)/debug-windows-32 + BR=$(BUILD_DIR)/release-windows-32 +else +ifeq ($(PLATFORM),darwin) + BD=$(BUILD_DIR)/debug-macos-32 + BR=$(BUILD_DIR)/release-macos-32 +else + BD=$(BUILD_DIR)/debug-$(PLATFORM)-32 + BR=$(BUILD_DIR)/release-$(PLATFORM)-32 +endif +endif +else +ifeq ($(PLATFORM),mingw32) + BD=$(BUILD_DIR)/debug-windows-$(ARCH) + BR=$(BUILD_DIR)/release-windows-$(ARCH) +else +ifeq ($(PLATFORM),darwin) + BD=$(BUILD_DIR)/debug-macos-$(ARCH) + BR=$(BUILD_DIR)/release-macos-$(ARCH) +else + BD=$(BUILD_DIR)/debug-$(PLATFORM)-$(ARCH) + BR=$(BUILD_DIR)/release-$(PLATFORM)-$(ARCH) +endif +endif +endif +endif + CDIR=$(MOUNT_DIR)/client SDIR=$(MOUNT_DIR)/server @@ -262,6 +310,8 @@ RCOMMONDIR=$(MOUNT_DIR)/renderercommon RGL1DIR=$(MOUNT_DIR)/renderergl1 RGL2DIR=$(MOUNT_DIR)/renderergl2 CMDIR=$(MOUNT_DIR)/qcommon +FSDIR=$(MOUNT_DIR)/filesystem +FSDIR_FSCORE=$(MOUNT_DIR)/filesystem/fscore SDLDIR=$(MOUNT_DIR)/sdl ASMDIR=$(MOUNT_DIR)/asm SYSDIR=$(MOUNT_DIR)/sys @@ -806,30 +856,41 @@ ifneq ($(BUILD_CLIENT),0) endif ifneq ($(BUILD_GAME_SO),0) - TARGETS += \ - $(B)/$(BASEGAME)/cgame$(SHLIBNAME) \ - $(B)/$(BASEGAME)/game$(SHLIBNAME) \ - $(B)/$(BASEGAME)/ui$(SHLIBNAME) +ifneq ($(BASEMOD), basemod) +TARGETS += \ + $(B)/$(BASEMOD)/cgame$(SHLIBNAME) \ + $(B)/$(BASEMOD)/core_game$(SHLIBNAME) \ + $(B)/$(BASEMOD)/ui$(SHLIBNAME) +else +TARGETS += \ + $(B)/$(BASEMOD)/cgame$(SHLIBNAME) \ + $(B)/$(BASEMOD)/game$(SHLIBNAME) \ + $(B)/$(BASEMOD)/ui$(SHLIBNAME) +endif endif ifneq ($(BUILD_GAME_QVM),0) - TARGETS += \ - $(B)/$(BASEGAME)/vm/cgame.qvm \ - $(B)/$(BASEGAME)/vm/game.qvm \ - $(B)/$(BASEGAME)/vm/ui.qvm \ - $(B)/$(BASEGAME)/vms-gpp-$(VERSION).pk3 +TARGETS += \ + $(B)/$(BASEMOD)/vm/cgame.qvm \ + $(B)/$(BASEMOD)/vm/game.qvm \ + $(B)/$(BASEMOD)/vm/ui.qvm \ + $(B)/$(BASEMOD)/vms-gpp-$(VERSION).pk3 endif ifneq ($(BUILD_GAME_QVM_11),0) - TARGETS += \ - $(B)/$(BASEGAME)_11/vm/cgame.qvm \ - $(B)/$(BASEGAME)_11/vm/ui.qvm \ - $(B)/$(BASEGAME)_11/vms-1.1.0-$(VERSION).pk3 +TARGETS += \ + $(B)/$(BASEMOD)_11/vm/cgame.qvm \ + $(B)/$(BASEMOD)_11/vm/ui.qvm \ + $(B)/$(BASEMOD)/vms-1.1-$(VERSION).pk3 endif ifneq ($(BUILD_DATA_PK3),0) - TARGETS += \ - $(B)/$(BASEGAME)/data-$(VERSION).pk3 +TARGETS += \ + $(B)/$(BASEMOD)/data-$(VERSION).pk3 +endif + +ifneq ($(NEW_FILESYSTEM),0) + BASE_CFLAGS += -DNEW_FILESYSTEM endif ifeq ($(USE_OPENAL),1) @@ -1229,7 +1290,7 @@ endif $(B).zip: $(TARGETS) ifeq ($(PLATFORM),darwin) - @("./make-macosx-app.sh" release $(ARCH); if [ "$$?" -eq 0 ] && [ -d "$(B)/Tremulous.app" ]; then rm -f $@; cd $(B) && zip --symlinks -r9 ../../$@ GPL COPYING CC `find "Tremulous.app" -print | sed -e "s!$(B)/!!g"`; else rm -f $@; cd $(B) && zip -r9 ../../$@ $(NAKED_TARGETS); fi) + @("./make-macosx-app.sh" release $(BASEMOD) $(ARCH); if [ "$$?" -eq 0 ] && [ -d "$(B)/Tremulous.app" ]; then rm -f $@; cd $(B) && zip --symlinks -r9 ../../$@ GPL COPYING CC `find "Tremulous.app" -print | sed -e "s!$(B)/!!g"`; else rm -f $@; cd $(B) && zip -r9 ../../$@ $(NAKED_TARGETS); fi) else @rm -f $@ @(cd $(B) && zip -r9 ../../$@ $(NAKED_TARGETS)) @@ -1261,8 +1322,9 @@ makedirs: @if [ ! -d $(B)/$(BASEGAME)/11/cgame ];then $(MKDIR) $(B)/$(BASEGAME)/11/cgame;fi @if [ ! -d $(B)/$(BASEGAME)/11/ui ];then $(MKDIR) $(B)/$(BASEGAME)/11/ui;fi @if [ ! -d $(B)/$(BASEGAME)/vm ];then $(MKDIR) $(B)/$(BASEGAME)/vm;fi - @if [ ! -d $(B)/$(BASEGAME)_11 ];then $(MKDIR) $(B)/$(BASEGAME)_11;fi - @if [ ! -d $(B)/$(BASEGAME)_11/vm ];then $(MKDIR) $(B)/$(BASEGAME)_11/vm;fi + @if [ ! -d $(B)/$(BASEMOD)_11 ];then $(MKDIR) $(B)/$(BASEMOD)_11;fi + @if [ ! -d $(B)/$(BASEMOD)_11/vm ];then $(MKDIR) $(B)/$(BASEMOD)_11/vm;fi + @if [ ! -d $(B)/$(BASEMOD) ];then $(MKDIR) $(B)/$(BASEMOD);fi @if [ ! -d $(B)/granger.dir ];then $(MKDIR) $(B)/granger.dir;fi @if [ ! -d $(B)/granger.dir/src ];then $(MKDIR) $(B)/granger.dir/src;fi @if [ ! -d $(B)/granger.dir/src/lua ];then $(MKDIR) $(B)/granger.dir/src/lua;fi @@ -1837,7 +1899,29 @@ Q3OBJ = \ $(B)/client/sdl_snd.o \ \ $(B)/client/con_log.o \ - $(B)/client/sys_main.o + $(B)/client/sys_main.o \ + \ + $(B)/client/fsc_cache.o \ + $(B)/client/fsc_crosshair.o \ + $(B)/client/fsc_gameparse.o \ + $(B)/client/fsc_iteration.o \ + $(B)/client/fsc_main.o \ + $(B)/client/fsc_md4.o \ + $(B)/client/fsc_misc.o \ + $(B)/client/fsc_os.o \ + $(B)/client/fsc_pk3.o \ + $(B)/client/fsc_sha256.o \ + $(B)/client/fsc_shader.o \ + $(B)/client/fs_commands.o \ + $(B)/client/fs_download.o \ + $(B)/client/fs_fileio.o \ + $(B)/client/fs_filelist.o \ + $(B)/client/fs_lookup.o \ + $(B)/client/fs_main.o \ + $(B)/client/fs_misc.o \ + $(B)/client/fs_tremulous.o \ + $(B)/client/fs_reference.o \ + $(B)/client/fs_trusted_vms.o ifdef MINGW Q3OBJ += \ @@ -2355,7 +2439,29 @@ Q3DOBJ = \ $(B)/ded/null_snddma.o \ \ $(B)/ded/con_log.o \ - $(B)/ded/sys_main.o + $(B)/ded/sys_main.o \ + \ + $(B)/ded/fsc_cache.o \ + $(B)/ded/fsc_crosshair.o \ + $(B)/ded/fsc_gameparse.o \ + $(B)/ded/fsc_iteration.o \ + $(B)/ded/fsc_main.o \ + $(B)/ded/fsc_md4.o \ + $(B)/ded/fsc_misc.o \ + $(B)/ded/fsc_os.o \ + $(B)/ded/fsc_pk3.o \ + $(B)/ded/fsc_sha256.o \ + $(B)/ded/fsc_shader.o \ + $(B)/ded/fs_commands.o \ + $(B)/ded/fs_download.o \ + $(B)/ded/fs_fileio.o \ + $(B)/ded/fs_filelist.o \ + $(B)/ded/fs_lookup.o \ + $(B)/ded/fs_main.o \ + $(B)/ded/fs_misc.o \ + $(B)/ded/fs_tremulous.o \ + $(B)/ded/fs_reference.o \ + $(B)/ded/fs_trusted_vms.o ifeq ($(ARCH),x86) Q3DOBJ += \ @@ -2455,11 +2561,11 @@ CGOBJ_ = \ CGOBJ = $(CGOBJ_) $(B)/$(BASEGAME)/cgame/cg_syscalls.o CGVMOBJ = $(CGOBJ_:%.o=%.asm) -$(B)/$(BASEGAME)/cgame$(SHLIBNAME): $(CGOBJ) +$(B)/$(BASEMOD)/cgame$(SHLIBNAME): $(CGOBJ) $(echo_cmd) "LD $@" $(Q)$(CC) $(SHLIBLDFLAGS) $(LDFLAGS) -o $@ $(CGOBJ) -$(B)/$(BASEGAME)/vm/cgame.qvm: $(CGVMOBJ) $(CGDIR)/cg_syscalls.asm $(Q3ASM) +$(B)/$(BASEMOD)/vm/cgame.qvm: $(CGVMOBJ) $(CGDIR)/cg_syscalls.asm $(Q3ASM) $(echo_cmd) "Q3ASM $@" $(Q)$(Q3ASM) -o $@ $(CGVMOBJ) $(CGDIR)/cg_syscalls.asm @@ -2504,7 +2610,7 @@ CGOBJ11_ = \ CGVMOBJ11 = $(CGOBJ11_:%.o=%.asm) -$(B)/$(BASEGAME)_11/vm/cgame.qvm: $(CGVMOBJ11) $(CGDIR)/cg_syscalls_11.asm $(Q3ASM) +$(B)/$(BASEMOD)_11/vm/cgame.qvm: $(CGVMOBJ11) $(CGDIR)/cg_syscalls_11.asm $(Q3ASM) $(echo_cmd) "Q3ASM_11 $@" $(Q)$(Q3ASM) -o $@ $(CGVMOBJ11) $(CGDIR)/cg_syscalls_11.asm @@ -2549,16 +2655,21 @@ GOBJ_ = \ GOBJ = $(GOBJ_) $(B)/$(BASEGAME)/game/g_syscalls.o GVMOBJ = $(GOBJ_:%.o=%.asm) -$(B)/$(BASEGAME)/game$(SHLIBNAME): $(GOBJ) +ifneq ($(BASEMOD), basemod) +$(B)/$(BASEMOD)/core_game$(SHLIBNAME): $(GOBJ) + $(echo_cmd) "LD $@" + $(Q)$(CC) $(SHLIBLDFLAGS) $(LDFLAGS) -o $@ $(GOBJ) +else +$(B)/$(BASEMOD)/game$(SHLIBNAME): $(GOBJ) $(echo_cmd) "LD $@" $(Q)$(CC) $(SHLIBLDFLAGS) $(LDFLAGS) -o $@ $(GOBJ) +endif -$(B)/$(BASEGAME)/vm/game.qvm: $(GVMOBJ) $(GDIR)/g_syscalls.asm $(Q3ASM) +$(B)/$(BASEMOD)/vm/game.qvm: $(GVMOBJ) $(GDIR)/g_syscalls.asm $(Q3ASM) $(echo_cmd) "Q3ASM $@" $(Q)$(Q3ASM) -o $@ $(GVMOBJ) $(GDIR)/g_syscalls.asm - ############################################################################# ## TREMULOUS UI ############################################################################# @@ -2579,10 +2690,14 @@ UIOBJ_ = \ UIOBJ = $(UIOBJ_) $(B)/$(BASEGAME)/ui/ui_syscalls.o UIVMOBJ = $(UIOBJ_:%.o=%.asm) -$(B)/$(BASEGAME)/ui$(SHLIBNAME): $(UIOBJ) +$(B)/$(BASEMOD)/ui$(SHLIBNAME): $(UIOBJ) $(echo_cmd) "LD $@" $(Q)$(CC) -I${ASSETS_DIR}/ui $(SHLIBLDFLAGS) $(LDFLAGS) -o $@ $(UIOBJ) +$(B)/$(BASEMOD)/vm/ui.qvm: $(UIVMOBJ) $(UIDIR)/ui_syscalls.asm $(Q3ASM) + $(echo_cmd) "Q3ASM $@" + $(Q)$(Q3ASM) -o $@ $(UIVMOBJ) $(UIDIR)/ui_syscalls.asm + $(B)/$(BASEGAME)/vm/ui.qvm: $(UIVMOBJ) $(UIDIR)/ui_syscalls.asm $(Q3ASM) $(echo_cmd) "Q3ASM $@" $(Q)$(Q3ASM) -o $@ $(UIVMOBJ) $(UIDIR)/ui_syscalls.asm @@ -2607,7 +2722,7 @@ UIVMOBJ11 = $(UIOBJ11_:%.o=%.asm) # XXX no dynamic library? -$(B)/$(BASEGAME)_11/vm/ui.qvm: $(UIVMOBJ11) $(UIDIR)/ui_syscalls_11.asm $(Q3ASM) +$(B)/$(BASEMOD)_11/vm/ui.qvm: $(UIVMOBJ11) $(UIDIR)/ui_syscalls_11.asm $(Q3ASM) $(echo_cmd) "Q3ASM $@" $(Q)$(Q3ASM) -o $@ $(UIVMOBJ11) $(UIDIR)/ui_syscalls_11.asm @@ -2615,20 +2730,25 @@ $(B)/$(BASEGAME)_11/vm/ui.qvm: $(UIVMOBJ11) $(UIDIR)/ui_syscalls_11.asm $(Q3ASM) ## QVM Package ############################################################################# -$(B)/$(BASEGAME)/vms-gpp-$(VERSION).pk3: $(B)/$(BASEGAME)/vm/ui.qvm $(B)/$(BASEGAME)/vm/cgame.qvm $(B)/$(BASEGAME)/vm/game.qvm - @(cd $(B)/$(BASEGAME) && zip -r vms-$(VERSION).pk3 vm/) - -$(B)/$(BASEGAME)_11/vms-1.1.0-$(VERSION).pk3: $(B)/$(BASEGAME)_11/vm/ui.qvm $(B)/$(BASEGAME)_11/vm/cgame.qvm - @(cd $(B)/$(BASEGAME)_11 && zip -r vms-$(VERSION).pk3 vm/) +ifneq ($(BUILD_GAME_QVM),0) +$(B)/$(BASEMOD)/vms-gpp-$(VERSION).pk3: $(B)/$(BASEMOD)/vm/ui.qvm $(B)/$(BASEMOD)/vm/cgame.qvm $(B)/$(BASEMOD)/vm/game.qvm + @(cd $(B)/$(BASEMOD) && zip -r vms-gpp-$(VERSION).pk3 vm/) +endif +ifneq ($(BUILD_GAME_QVM_11),0) +$(B)/$(BASEMOD)/vms-1.1-$(VERSION).pk3: $(B)/$(BASEMOD)_11/vm/ui.qvm $(B)/$(BASEMOD)_11/vm/cgame.qvm + @(cd $(B)/$(BASEMOD)_11 && zip -r vms-1.1-$(VERSION).pk3 vm/) +endif ############################################################################# ## Assets Package ############################################################################# -$(B)/$(BASEGAME)/data-$(VERSION).pk3: $(ASSETS_DIR)/ui/main.menu +ifneq ($(BUILD_DATA_PK3),0) +$(B)/$(BASEMOD)/data-$(VERSION).pk3: $(ASSETS_DIR)/ui/main.menu @(cd $(ASSETS_DIR) && zip -r data-$(VERSION).pk3 *) - @mv $(ASSETS_DIR)/data-$(VERSION).pk3 $(B)/$(BASEGAME) + @mv $(ASSETS_DIR)/data-$(VERSION).pk3 $(B)/$(BASEMOD) +endif ############################################################################# ## CLIENT/SERVER RULES @@ -2650,6 +2770,11 @@ $(B)/client/%.o: $(SDIR)/%.c $(B)/client/%.o: $(CMDIR)/%.c $(DO_CC) +$(B)/client/%.o: $(FSDIR)/%.cpp + $(DO_CXX) +$(B)/client/%.o: $(FSDIR_FSCORE)/%.cpp + $(DO_CXX) + $(B)/client/%.o: $(CDIR)/%.cpp $(DO_CXX) @@ -2758,6 +2883,11 @@ $(B)/ded/%.o: $(SDIR)/%.cpp $(B)/ded/%.o: $(CMDIR)/%.c $(DO_DED_CC) +$(B)/ded/%.o: $(FSDIR)/%.cpp + $(DO_DED_CXX) +$(B)/ded/%.o: $(FSDIR_FSCORE)/%.cpp + $(DO_DED_CXX) + $(B)/ded/%.o: $(CMDIR)/%.cpp $(DO_DED_CXX) @@ -2814,13 +2944,13 @@ $(B)/$(BASEGAME)/cgame/%.asm: $(CGDIR)/%.c $(Q3LCC) $(DO_CGAME_Q3LCC) # CGAME (1.1 COMPATIBLE) -#$(B)/$(BASEGAME)_11/cgame/bg_%.o: $(GDIR)/bg_%.c +#$(B)/$(BASEMOD)_11/cgame/bg_%.o: $(GDIR)/bg_%.c # $(DO_CGAME_CC_11) # -#$(B)/$(BASEGAME)_11/cgame/ui_%.o: $(UIDIR)/ui_%.c +#$(B)/$(BASEMOD)_11/cgame/ui_%.o: $(UIDIR)/ui_%.c # $(DO_CGAME_CC_11) # -#$(B)/$(BASEGAME)_11/cgame/%.o: $(CGDIR)/%.c +#$(B)/$(BASEMOD)_11/cgame/%.o: $(CGDIR)/%.c # $(DO_CGAME_CC_11) $(B)/$(BASEGAME)/11/cgame/%.asm: $(CGDIR)/%.c $(Q3LCC) @@ -2914,14 +3044,203 @@ ifneq ($(B),) -include $(OBJ_D_FILES) $(TOOLSOBJ_D_FILES) endif +ifneq ($(BUILD_GAME_QVM),0) +ifneq ($(BUILD_GAME_QVM_11),0) +ifneq ($(BUILD_DATA_PK3),0) +ifneq ($(BASEMOD), basemod) +.PHONY: all clean clean2 clean-debug clean-release \ + debug default dist distclean makedirs release targets \ + toolsclean toolsclean2 toolsclean-debug toolsclean-release \ + $(OBJ_D_FILES) $(TOOLSOBJ_D_FILES) $(B)/scripts \ + $(B)/$(BASEMOD)/data-$(VERSION).pk3 \ + $(B)/$(BASEMOD)/vms-1.1-$(VERSION).pk3 \ + $(B)/$(BASEMOD)/vms-gpp-$(VERSION).pk3 \ + $(B)/$(BASEMOD)/cgame$(SHLIBNAME) \ + $(B)/$(BASEMOD)/core_game$(SHLIBNAME) \ + $(B)/$(BASEMOD)/ui$(SHLIBNAME) \ + $(B).zip +else .PHONY: all clean clean2 clean-debug clean-release \ debug default dist distclean makedirs release targets \ toolsclean toolsclean2 toolsclean-debug toolsclean-release \ $(OBJ_D_FILES) $(TOOLSOBJ_D_FILES) $(B)/scripts \ - $(B)/$(BASEGAME)/data-$(VERSION).pk3 \ - $(B)/$(BASEGAME)_11/vms-$(VERSION).pk3 \ - $(B)/$(BASEGAME)/vms-$(VERSION).pk3 \ + $(B)/$(BASEMOD)/data-$(VERSION).pk3 \ + $(B)/$(BASEMOD)/vms-1.1-$(VERSION).pk3 \ + $(B)/$(BASEMOD)/vms-gpp-$(VERSION).pk3 \ + $(B)/$(BASEMOD)/cgame$(SHLIBNAME) \ + $(B)/$(BASEMOD)/game$(SHLIBNAME) \ + $(B)/$(BASEMOD)/ui$(SHLIBNAME) \ $(B).zip +endif +else +ifneq ($(BASEMOD), basemod) +.PHONY: all clean clean2 clean-debug clean-release \ + debug default dist distclean makedirs release targets \ + toolsclean toolsclean2 toolsclean-debug toolsclean-release \ + $(OBJ_D_FILES) $(TOOLSOBJ_D_FILES) $(B)/scripts \ + $(B)/$(BASEMOD)/vms-1.1-$(VERSION).pk3 \ + $(B)/$(BASEMOD)/vms-gpp-$(VERSION).pk3 \ + $(B)/$(BASEMOD)/cgame$(SHLIBNAME) \ + $(B)/$(BASEMOD)/core_game$(SHLIBNAME) \ + $(B)/$(BASEMOD)/ui$(SHLIBNAME) \ + $(B).zip +else +.PHONY: all clean clean2 clean-debug clean-release \ + debug default dist distclean makedirs release targets \ + toolsclean toolsclean2 toolsclean-debug toolsclean-release \ + $(OBJ_D_FILES) $(TOOLSOBJ_D_FILES) $(B)/scripts \ + $(B)/$(BASEMOD)/vms-1.1-$(VERSION).pk3 \ + $(B)/$(BASEMOD)/vms-gpp-$(VERSION).pk3 \ + $(B)/$(BASEMOD)/cgame$(SHLIBNAME) \ + $(B)/$(BASEMOD)/game$(SHLIBNAME) \ + $(B)/$(BASEMOD)/ui$(SHLIBNAME) \ + $(B).zip +endif +endif +else +ifneq ($(BUILD_DATA_PK3),0) +ifneq ($(BASEMOD), basemod) +.PHONY: all clean clean2 clean-debug clean-release \ + debug default dist distclean makedirs release targets \ + toolsclean toolsclean2 toolsclean-debug toolsclean-release \ + $(OBJ_D_FILES) $(TOOLSOBJ_D_FILES) $(B)/scripts \ + $(B)/$(BASEMOD)/data-$(VERSION).pk3 \ + $(B)/$(BASEMOD)/vms-gpp-$(VERSION).pk3 \ + $(B)/$(BASEMOD)/cgame$(SHLIBNAME) \ + $(B)/$(BASEMOD)/core_game$(SHLIBNAME) \ + $(B)/$(BASEMOD)/ui$(SHLIBNAME) \ + $(B).zip +else +.PHONY: all clean clean2 clean-debug clean-release \ + debug default dist distclean makedirs release targets \ + toolsclean toolsclean2 toolsclean-debug toolsclean-release \ + $(OBJ_D_FILES) $(TOOLSOBJ_D_FILES) $(B)/scripts \ + $(B)/$(BASEMOD)/data-$(VERSION).pk3 \ + $(B)/$(BASEMOD)/vms-gpp-$(VERSION).pk3 \ + $(B)/$(BASEMOD)/cgame$(SHLIBNAME) \ + $(B)/$(BASEMOD)/game$(SHLIBNAME) \ + $(B)/$(BASEMOD)/ui$(SHLIBNAME) \ + $(B).zip +endif +else +ifneq ($(BASEMOD), basemod) +.PHONY: all clean clean2 clean-debug clean-release \ + debug default dist distclean makedirs release targets \ + toolsclean toolsclean2 toolsclean-debug toolsclean-release \ + $(OBJ_D_FILES) $(TOOLSOBJ_D_FILES) $(B)/scripts \ + $(B)/$(BASEMOD)/vms-gpp-$(VERSION).pk3 \ + $(B)/$(BASEMOD)/cgame$(SHLIBNAME) \ + $(B)/$(BASEMOD)/core_game$(SHLIBNAME) \ + $(B)/$(BASEMOD)/ui$(SHLIBNAME) \ + $(B).zip +else +.PHONY: all clean clean2 clean-debug clean-release \ + debug default dist distclean makedirs release targets \ + toolsclean toolsclean2 toolsclean-debug toolsclean-release \ + $(OBJ_D_FILES) $(TOOLSOBJ_D_FILES) $(B)/scripts \ + $(B)/$(BASEMOD)/vms-gpp-$(VERSION).pk3 \ + $(B)/$(BASEMOD)/cgame$(SHLIBNAME) \ + $(B)/$(BASEMOD)/game$(SHLIBNAME) \ + $(B)/$(BASEMOD)/ui$(SHLIBNAME) \ + $(B).zip +endif +endif +endif +else +ifneq ($(BUILD_GAME_QVM_11),0) +ifneq ($(BUILD_DATA_PK3),0) +ifneq ($(BASEMOD), basemod) +.PHONY: all clean clean2 clean-debug clean-release \ + debug default dist distclean makedirs release targets \ + toolsclean toolsclean2 toolsclean-debug toolsclean-release \ + $(OBJ_D_FILES) $(TOOLSOBJ_D_FILES) $(B)/scripts \ + $(B)/$(BASEMOD)/data-$(VERSION).pk3 \ + $(B)/$(BASEMOD)/vms-1.1-$(VERSION).pk3 \ + $(B)/$(BASEMOD)/cgame$(SHLIBNAME) \ + $(B)/$(BASEMOD)/core_game$(SHLIBNAME) \ + $(B)/$(BASEMOD)/ui$(SHLIBNAME) \ + $(B).zip +else +.PHONY: all clean clean2 clean-debug clean-release \ + debug default dist distclean makedirs release targets \ + toolsclean toolsclean2 toolsclean-debug toolsclean-release \ + $(OBJ_D_FILES) $(TOOLSOBJ_D_FILES) $(B)/scripts \ + $(B)/$(BASEMOD)/data-$(VERSION).pk3 \ + $(B)/$(BASEMOD)/vms-1.1-$(VERSION).pk3 \ + $(B)/$(BASEMOD)/cgame$(SHLIBNAME) \ + $(B)/$(BASEMOD)/game$(SHLIBNAME) \ + $(B)/$(BASEMOD)/ui$(SHLIBNAME) \ + $(B).zip +endif +else +ifneq ($(BASEMOD), basemod) +.PHONY: all clean clean2 clean-debug clean-release \ + debug default dist distclean makedirs release targets \ + toolsclean toolsclean2 toolsclean-debug toolsclean-release \ + $(OBJ_D_FILES) $(TOOLSOBJ_D_FILES) $(B)/scripts \ + $(B)/$(BASEMOD)/vms-1.1-$(VERSION).pk3 \ + $(B)/$(BASEMOD)/cgame$(SHLIBNAME) \ + $(B)/$(BASEMOD)/core_game$(SHLIBNAME) \ + $(B)/$(BASEMOD)/ui$(SHLIBNAME) \ + $(B).zip +else +.PHONY: all clean clean2 clean-debug clean-release \ + debug default dist distclean makedirs release targets \ + toolsclean toolsclean2 toolsclean-debug toolsclean-release \ + $(OBJ_D_FILES) $(TOOLSOBJ_D_FILES) $(B)/scripts \ + $(B)/$(BASEMOD)/vms-1.1-$(VERSION).pk3 \ + $(B)/$(BASEMOD)/cgame$(SHLIBNAME) \ + $(B)/$(BASEMOD)/game$(SHLIBNAME) \ + $(B)/$(BASEMOD)/ui$(SHLIBNAME) \ + $(B).zip +endif +endif +else +ifneq ($(BUILD_DATA_PK3),0) +ifneq ($(BASEMOD), basemod) +.PHONY: all clean clean2 clean-debug clean-release \ + debug default dist distclean makedirs release targets \ + toolsclean toolsclean2 toolsclean-debug toolsclean-release \ + $(OBJ_D_FILES) $(TOOLSOBJ_D_FILES) $(B)/scripts \ + $(B)/$(BASEMOD)/data-$(VERSION).pk3 \ + $(B)/$(BASEMOD)/cgame$(SHLIBNAME) \ + $(B)/$(BASEMOD)/core_game$(SHLIBNAME) \ + $(B)/$(BASEMOD)/ui$(SHLIBNAME) \ + $(B).zip +else +.PHONY: all clean clean2 clean-debug clean-release \ + debug default dist distclean makedirs release targets \ + toolsclean toolsclean2 toolsclean-debug toolsclean-release \ + $(OBJ_D_FILES) $(TOOLSOBJ_D_FILES) $(B)/scripts \ + $(B)/$(BASEMOD)/data-$(VERSION).pk3 \ + $(B)/$(BASEMOD)/cgame$(SHLIBNAME) \ + $(B)/$(BASEMOD)/game$(SHLIBNAME) \ + $(B)/$(BASEMOD)/ui$(SHLIBNAME) \ + $(B).zip +endif +else +ifneq ($(BASEMOD), basemod) +.PHONY: all clean clean2 clean-debug clean-release \ + debug default dist distclean makedirs release targets \ + toolsclean toolsclean2 toolsclean-debug toolsclean-release \ + $(OBJ_D_FILES) $(TOOLSOBJ_D_FILES) $(B)/scripts \ + $(B)/$(BASEMOD)/cgame$(SHLIBNAME) \ + $(B)/$(BASEMOD)/core_game$(SHLIBNAME) \ + $(B)/$(BASEMOD)/ui$(SHLIBNAME) \ + $(B).zip +else +.PHONY: all clean clean2 clean-debug clean-release \ + debug default dist distclean makedirs release targets \ + toolsclean toolsclean2 toolsclean-debug toolsclean-release \ + $(OBJ_D_FILES) $(TOOLSOBJ_D_FILES) $(B)/scripts \ + $(B)/$(BASEMOD)/cgame$(SHLIBNAME) \ + $(B)/$(BASEMOD)/game$(SHLIBNAME) \ + $(B)/$(BASEMOD)/ui$(SHLIBNAME) \ + $(B).zip +endif +endif +endif +endif # If the target name contains "clean", don't do a parallel build ifneq ($(findstring clean, $(MAKECMDGOALS)),) diff --git a/README.md b/README.md index a40301c01..9219abc1f 100644 --- a/README.md +++ b/README.md @@ -9,9 +9,10 @@ You can install the latest released binaries from GrangerHub's release page, following these steps: * Download the most recent .zip file for your platform from https://github.com/GrangerHub/tremulous/releases - - The 64 bit Windows release would be named release-mingw32-x86_64.zip - - The 64 bit Linux release would be named release-linux-x86_64.zip - - The 64 bit Mac OS X release would be named release-darwin-x86_64.zip + - The 64 bit Windows release would be named release-windows-64.zip + - The 32 bit Windows release would be named release-windows-32.zip + - The 64 bit Linux release would be named release-linux-64.zip + - The 64 bit Mac OS X release would be named release-macos-64.zip * Unzip the release .zip anywhere * Run the tremulous.exe from the unzipped release directory. - When you run the tremulous.exe binary for the first time, it may go through a bootstrap (download needed assets, generate an RSA key) process which may take a few minutes. @@ -65,8 +66,8 @@ For Linux and Mac OS X builds, follow these steps. git clone https://github.com/GrangerHub/tremulous.git cd tremulous make -# cd build/release-darwin-x86_64/ -# cd build/release-linux-x86_64/ +# cd build/release-macos-64/ +# cd build/release-linux-64/ ./tremulous ``` diff --git a/assets/ui/folders.menu b/assets/ui/folders.menu index bda693618..3fbb180fc 100644 --- a/assets/ui/folders.menu +++ b/assets/ui/folders.menu @@ -102,6 +102,24 @@ } } + itemDef + { + text "Browse basemod Folder" + type ITEM_TYPE_BUTTON + textscale .25 + style WINDOW_STYLE_EMPTY + rect X (Y+(3*(BUTT_H+ELEM_GAP))) W BUTT_H + textalign ALIGN_CENTER + textvalign VALIGN_CENTER + forecolor 1 1 1 1 + visible MENU_TRUE + action + { + play "sound/misc/menu1.wav"; + exec browseBasemod + } + } + itemDef { text "OK" diff --git a/default-assets b/default-assets new file mode 160000 index 000000000..14b5732e8 --- /dev/null +++ b/default-assets @@ -0,0 +1 @@ +Subproject commit 14b5732e839265a0544c7e2bba16f2c50bd85a59 diff --git a/external/nettle-3.3/CMakeLists.txt b/external/nettle-3.3/CMakeLists.txt index 72c7f86a9..ef720a6b8 100644 --- a/external/nettle-3.3/CMakeLists.txt +++ b/external/nettle-3.3/CMakeLists.txt @@ -34,5 +34,4 @@ include_directories ( include ) -# command: /usr/bin/clang -Wall -Wextra -DLUA_COMPAT_5_2 -fPIC -fpic -o build/release-darwin-x86_64/nettle/bignum.o -c src/nettle-3.3/nettle/bignum.c - +# command: /usr/bin/clang -Wall -Wextra -DLUA_COMPAT_5_2 -fPIC -fpic -o build/release-macos-64/nettle/bignum.o -c src/nettle-3.3/nettle/bignum.c diff --git a/make-macosx-app.sh b/make-macosx-app.sh index 05a72bba1..1e78d9a5e 100755 --- a/make-macosx-app.sh +++ b/make-macosx-app.sh @@ -3,9 +3,9 @@ # Let's make the user give us a target to work with. # architecture is assumed universal if not specified, and is optional. # if arch is defined, it we will store the .app bundle in the target arch build directory -if [ $# == 0 ] || [ $# -gt 2 ]; then - echo "Usage: $0 target " - echo "Example: $0 release x86" +if [ $# == 0 ] || [ $# -gt 3 ]; then + echo "Usage: $0 target " + echo "Example: $0 release basemod x86" echo "Valid targets are:" echo " release" echo " debug" @@ -33,13 +33,13 @@ fi CURRENT_ARCH="" # validate the architecture if it was specified -if [ "$2" != "" ]; then - if [ "$2" == "x86" ]; then +if [ "$3" != "" ]; then + if [ "$3" == "x86" ]; then CURRENT_ARCH="x86" - elif [ "$2" == "x86_64" ]; then + elif [ "$3" == "x86_64" ]; then CURRENT_ARCH="x86_64" else - echo "Invalid architecture: $2" + echo "Invalid architecture: $3" echo "Valid architectures are:" echo " x86" echo " x86_64" @@ -127,11 +127,15 @@ IOQ3_MP_CGAME_ARCHS="" IOQ3_MP_GAME_ARCHS="" IOQ3_MP_UI_ARCHS="" -BASEDIR="gpp" +BASEDIR=$2 CGAME="cgame" -GAME="game" UI="ui" +if [[ "${2}" == "basemod" ]]; then + GAME="game" +else + GAME="core_game" +fi RENDERER_OPENGL="renderer_opengl" @@ -142,7 +146,8 @@ GAME_NAME="${GAME}.dylib" UI_NAME="${UI}.dylib" GIT_REV=$(git describe --tags) -VMS_PK3="vms-${GIT_REV}.pk3" +VMS_GPP_PK3="vms-gpp-${GIT_REV}.pk3" +VMS_11_PK3="vms-1.1-${GIT_REV}.pk3" DATA_PK3="data-${GIT_REV}.pk3" RENDERER_OPENGL1_NAME="${RENDERER_OPENGL}1.dylib" @@ -153,7 +158,7 @@ ICNS="Tremulous.icns" PKGINFO="APPLIOQ3" OBJROOT="build" -#BUILT_PRODUCTS_DIR="${OBJROOT}/${TARGET_NAME}-darwin-${CURRENT_ARCH}" +#BUILT_PRODUCTS_DIR="${OBJROOT}/${TARGET_NAME}-macos-${CURRENT_ARCH}" PRODUCT_NAME="Tremulous" WRAPPER_EXTENSION="app" WRAPPER_NAME="${PRODUCT_NAME}.${WRAPPER_EXTENSION}" @@ -165,7 +170,13 @@ EXECUTABLE_NAME="tremulous" # loop through the architectures to build string lists for each universal binary for ARCH in $SEARCH_ARCHS; do CURRENT_ARCH=${ARCH} - BUILT_PRODUCTS_DIR="${OBJROOT}/${TARGET_NAME}-darwin-${CURRENT_ARCH}" + if [ "${CURRENT_ARCH}" == "x86_64" ]; then + BUILT_PRODUCTS_DIR="${OBJROOT}/${TARGET_NAME}-macos-64" + elif [[ "${CURRENT_ARCH}" == "x86" ]]; then + BUILT_PRODUCTS_DIR="${OBJROOT}/${TARGET_NAME}-macos-32" + else + BUILT_PRODUCTS_DIR="${OBJROOT}/${TARGET_NAME}-macos-${CURRENT_ARCH}" + fi IOQ3_CLIENT="${EXECUTABLE_NAME}" IOQ3_SERVER="${DEDICATED_NAME}" IOQ3_RENDERER_GL1="${RENDERER_OPENGL}1.dylib" @@ -227,13 +238,20 @@ if [ "${IOQ3_CLIENT_ARCHS}" == "" ]; then fi # set the final application bundle output directory -if [ "${2}" == "" ]; then - BUILT_PRODUCTS_DIR="${OBJROOT}/${TARGET_NAME}-darwin-universal" +if [ "${3}" == "" ]; then + BUILT_PRODUCTS_DIR="${OBJROOT}/${TARGET_NAME}-macos-universal" if [ ! -d ${BUILT_PRODUCTS_DIR} ]; then mkdir -p ${BUILT_PRODUCTS_DIR} || exit 1; fi else - BUILT_PRODUCTS_DIR="${OBJROOT}/${TARGET_NAME}-darwin-${CURRENT_ARCH}" + if [ "${CURRENT_ARCH}" == "x86_64" ]; then + BUILT_PRODUCTS_DIR="${OBJROOT}/${TARGET_NAME}-macos-64" + elif [[ "${CURRENT_ARCH}" == "x86" ]]; then + BUILT_PRODUCTS_DIR="${OBJROOT}/${TARGET_NAME}-macos-32" + else + BUILT_PRODUCTS_DIR="${OBJROOT}/${TARGET_NAME}-macos-${CURRENT_ARCH}" + fi + #statements fi BUNDLEBINDIR="${BUILT_PRODUCTS_DIR}/${EXECUTABLE_FOLDER_PATH}" @@ -251,6 +269,12 @@ echo "" if [ ! -d ${BUILT_PRODUCTS_DIR}/${EXECUTABLE_FOLDER_PATH}/$BASEDIR ]; then mkdir -p ${BUILT_PRODUCTS_DIR}/${EXECUTABLE_FOLDER_PATH}/$BASEDIR || exit 1; fi +if [ ! -d ${BUILT_PRODUCTS_DIR}/${EXECUTABLE_FOLDER_PATH}/gpp ]; then + mkdir -p ${BUILT_PRODUCTS_DIR}/${EXECUTABLE_FOLDER_PATH}/gpp || exit 1; +fi +if [ ! -d ${BUILT_PRODUCTS_DIR}/${EXECUTABLE_FOLDER_PATH}/base ]; then + mkdir -p ${BUILT_PRODUCTS_DIR}/${EXECUTABLE_FOLDER_PATH}/base || exit 1; +fi if [ ! -d ${BUILT_PRODUCTS_DIR}/${UNLOCALIZED_RESOURCES_FOLDER_PATH} ]; then mkdir -p ${BUILT_PRODUCTS_DIR}/${UNLOCALIZED_RESOURCES_FOLDER_PATH} || exit 1; fi @@ -258,7 +282,8 @@ fi # copy and generate some application bundle resources cp external/libs/macosx/*.dylib ${BUILT_PRODUCTS_DIR}/${EXECUTABLE_FOLDER_PATH} cp ${ICNSDIR}/${ICNS} ${BUILT_PRODUCTS_DIR}/${UNLOCALIZED_RESOURCES_FOLDER_PATH}/$ICNS || exit 1; -cp "${BUILT_PRODUCTS_DIR}/${BASEDIR}/${VMS_PK3}" "${BUNDLEBINDIR}/${BASEDIR}/${VMS_PK3}" +cp "${BUILT_PRODUCTS_DIR}/${BASEDIR}/${VMS_GPP_PK3}" "${BUNDLEBINDIR}/${BASEDIR}/${VMS_GPP_PK3}" +cp "${BUILT_PRODUCTS_DIR}/${BASEDIR}/${VMS_11_PK3}" "${BUNDLEBINDIR}/${BASEDIR}/${VMS_11_PK3}" cp "${BUILT_PRODUCTS_DIR}/${BASEDIR}/${DATA_PK3}" "${BUNDLEBINDIR}/${BASEDIR}/${DATA_PK3}" echo -n ${PKGINFO} > ${BUILT_PRODUCTS_DIR}/${CONTENTS_FOLDER_PATH}/PkgInfo || exit 1; echo " @@ -290,7 +315,7 @@ echo " LSMinimumSystemVersion ${MACOSX_DEPLOYMENT_TARGET} NSHumanReadableCopyright - Copyright © 1999-2015 Id Software LLC, a ZeniMax Media company, ioquake3, Darklegion Development, GrangerHub, and Tremulous community contributors. + Copyright © 1999-2020 Id Software LLC, a ZeniMax Media company, ioquake3, Darklegion Development, GrangerHub, and Tremulous community contributors. NSPrincipalClass NSApplication diff --git a/misc/download-paks.sh b/misc/download-paks.sh index 8da3c8578..0db10bfea 100755 --- a/misc/download-paks.sh +++ b/misc/download-paks.sh @@ -1,52 +1,104 @@ #!/bin/bash -URL="https://github.com/wtfbbqhax/tremulous-data/raw/master/" - -packages=" -data-1.1.0.pk3 -data-gpp1.pk3 -map-arachnid2-1.1.0.pk3 -map-atcs-1.1.0.pk3 -map-karith-1.1.0.pk3 -map-nexus6-1.1.0.pk3 -map-niveus-1.1.0.pk3 -map-transit-1.1.0.pk3 -map-tremor-1.1.0.pk3 -map-uncreation-1.1.0.pk3 -" - -for dir in ./build/*; do +REPO_ROOT="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )/.." + +cd $REPO_ROOT + +default_assets_path="${REPO_ROOT}/default-assets" + +# prepare the default-assets repo +git submodule init +git submodule update + +for dir in ${REPO_ROOT}/build/*; do if [[ ! -d $dir ]]; then continue; fi - # Download + # Copy files + + if [[ $dir == "${REPO_ROOT}/build/release-macos-64" ]]; then + pushd $dir/Tremulous.app/Contents/MacOS/base/ + else + mkdir -pv $dir/base + pushd $dir/base + fi + + if [[ $dir == "${REPO_ROOT}/build/release-macos-64" ]]; then + for i in $default_assets_path/base/*.pk3; do + cp $i "${PWD}/" + done + else + cp -av $default_assets_path/base/*.pk3 ./ + fi + + popd - if [[ $dir == "./build/release-darwin-x86_64" ]]; then + if [[ $dir == "${REPO_ROOT}/build/release-macos-64" ]]; then pushd $dir/Tremulous.app/Contents/MacOS/gpp/ else + mkdir -pv $dir/gpp pushd $dir/gpp fi - for i in $packages; do - if [[ -e $package ]]; then - rm -f $package # only want 1 copy - fi - curl -OL $URL/$i - done + if [[ $dir == "${REPO_ROOT}/build/release-macos-64" ]]; then + for i in $default_assets_path/gpp/*.pk3; do + cp $i "${PWD}/" + done + else + cp -av $default_assets_path/gpp/*.pk3 ./ + fi popd - # Repackage + if [[ $dir == "${REPO_ROOT}/build/release-macos-64" ]]; then + pushd $dir/Tremulous.app/Contents/MacOS/trem13/ + else + mkdir -pv $dir/trem13 + pushd $dir/trem13 + fi + + mv *.pk3 $dir/ + if [[ $dir == "${REPO_ROOT}/build/release-macos-64" ]]; then + for i in $default_assets_path/trem13/*.pk3; do + cp $i "${PWD}/" + done + else + cp -a $default_assets_path/trem13/*.pk3 ./ + fi + + popd + + # Repackage and remove the non-default pk3 files from trem13/ pushd $dir - if [[ $dir == "./build/release-darwin-x86_64" ]]; then - zip -r ../$(basename $dir).zip Tremulous.app + if [[ $dir == "${REPO_ROOT}/build/release-macos-64" ]]; then + zip -d ${REPO_ROOT}/build/$(basename $dir).zip "Tremulous.app/Contents/MacOS/trem13/*.pk3" + zip -r ${REPO_ROOT}/build/$(basename $dir).zip Tremulous.app else - zip -r ../$(basename $dir).zip gpp/*.pk3 + zip -r ${REPO_ROOT}/build/$(basename $dir).zip base/*.pk3 + zip -r ${REPO_ROOT}/build/$(basename $dir).zip gpp/*.pk3 + zip -d ${REPO_ROOT}/build/$(basename $dir).zip "trem13/*.pk3" + zip -r ${REPO_ROOT}/build/$(basename $dir).zip trem13/*.pk3 fi popd -done + if [[ $dir == "${REPO_ROOT}/build/release-macos-64" ]]; then + pushd $dir/Tremulous.app/Contents/MacOS/trem13/ + else + pushd $dir/trem13 + fi + + if [[ $dir == "${REPO_ROOT}/build/release-macos-64" ]]; then + for i in $dir/*.pk3; do + cp $i "${PWD}/" + done + else + cp -an $dir/*.pk3 ./ + fi + rm -rf $dir/*.pk3 + + popd +done diff --git a/misc/make-macosx-ub.sh b/misc/make-macosx-ub.sh index 08bd99444..435249e02 100755 --- a/misc/make-macosx-ub.sh +++ b/misc/make-macosx-ub.sh @@ -67,28 +67,28 @@ fi NCPU=`sysctl -n hw.ncpu` # x86_64 client and server -#if [ -d build/release-release-x86_64 ]; then -# rm -r build/release-darwin-x86_64 +#if [ -d build/release-macos-64 ]; then +# rm -r build/release-macos-64 #fi (ARCH=x86_64 CC=gcc-4.0 CFLAGS=$X86_64_CFLAGS LDFLAGS=$X86_64_LDFLAGS make -j$NCPU) || exit 1; echo;echo # x86 client and server -#if [ -d build/release-darwin-x86 ]; then -# rm -r build/release-darwin-x86 +#if [ -d build/release-macos-32 ]; then +# rm -r build/release-macos-32 #fi (ARCH=x86 CC=gcc-4.0 CFLAGS=$X86_CFLAGS LDFLAGS=$X86_LDFLAGS make -j$NCPU) || exit 1; echo;echo # PPC client and server -#if [ -d build/release-darwin-ppc ]; then -# rm -r build/release-darwin-ppc +#if [ -d build/release-macos-ppc ]; then +# rm -r build/release-macos-ppc #fi (ARCH=ppc CC=gcc-4.0 CFLAGS=$PPC_CFLAGS LDFLAGS=$PPC_LDFLAGS make -j$NCPU) || exit 1; echo # use the following shell script to build a universal application bundle -"./make-macosx-app.sh" release +"./make-macosx-app.sh" release basemod diff --git a/misc/make-macosx.sh b/misc/make-macosx.sh index 7d48736cf..c6533accc 100755 --- a/misc/make-macosx.sh +++ b/misc/make-macosx.sh @@ -30,7 +30,14 @@ if [ -z "$DARWIN_GCC_ARCH" ]; then fi CC=gcc-4.0 -DESTDIR=build/release-darwin-${BUILDARCH} +if [ "${BUILDARCH}" == "x86_64" ]; then + DESTDIR=build/release-macos-64 +elif [[ "${BUILDARCH}" == "x86" ]]; then + DESTDIR=build/release-macos-32 +else + DESTDIR=build/release-macos-${BUILDARCH} +fi + #statements cd `dirname $0` if [ ! -f Makefile ]; then @@ -68,10 +75,10 @@ NCPU=`sysctl -n hw.ncpu` # intel client and server -#if [ -d build/release-darwin-${BUILDARCH} ]; then -# rm -r build/release-darwin-${BUILDARCH} +#if [ -d build/release-macos-${BUILDARCH} ]; then +# rm -r build/release-macos-${BUILDARCH} #fi (ARCH=${BUILDARCH} CFLAGS=$ARCH_CFLAGS LDFLAGS=$ARCH_LDFLAGS make -j$NCPU) || exit 1; # use the following shell script to build an application bundle -"./make-macosx-app.sh" release ${BUILDARCH} +"./make-macosx-app.sh" release basemod ${BUILDARCH} diff --git a/misc/travis-ci-build.sh b/misc/travis-ci-build.sh index 9fc9bd02b..9dcaba997 100755 --- a/misc/travis-ci-build.sh +++ b/misc/travis-ci-build.sh @@ -5,8 +5,17 @@ failed=0 if [[ $PLATFORM == "darwin" ]]; then rm -rf build - USE_FREETYPE=0 USE_RESTCLIENT=1 USE_INTERNAL_LUA=1 make -j 2 release || failed=1 + BASEMOD=trem13 USE_FREETYPE=0 USE_RESTCLIENT=1 USE_INTERNAL_LUA=1 make -j 2 release || failed=1 ./misc/download-paks.sh + REPO_ROOT="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )/.." + for dir in ${REPO_ROOT}/build/*; do + if [[ ! -d $dir ]]; then + continue; + fi + + zip -d ${REPO_ROOT}/build/$(basename $dir).zip "Tremulous.app/Contents/MacOS/trem13/cgame.*" + zip -d ${REPO_ROOT}/build/$(basename $dir).zip "Tremulous.app/Contents/MacOS/trem13/ui.*" + done fi if [[ $failed -eq 1 ]]; then @@ -16,4 +25,3 @@ else fi exit $failed - diff --git a/misc/travis-ci-docker-build.sh b/misc/travis-ci-docker-build.sh new file mode 100755 index 000000000..18e33ae3c --- /dev/null +++ b/misc/travis-ci-docker-build.sh @@ -0,0 +1,27 @@ +#!/bin/bash +failed=0 + +BASEMOD=trem13 USE_RESTCLIENT=1 USE_INTERNAL_LUA=1 make -j 2 || failed=1 + +if [[ $failed -eq 1 ]]; then + echo "Build failure." + exit $failed +fi + +./misc/download-paks.sh +REPO_ROOT="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )/.." +P=$(uname | sed -e 's/_.*//' | tr '[:upper:]' '[:lower:]' | sed -e 's/\//_/g') +PLATFORM=${PLATFORM:-$P} +for dir in ${REPO_ROOT}/build/*; do + if [[ ! -d $dir ]]; then + continue; + fi + + if [[ $PLATFORM != "darwin" ]]; then + zip -d ${REPO_ROOT}/build/$(basename $dir).zip "trem13_11/vm/*" + zip -d ${REPO_ROOT}/build/$(basename $dir).zip "trem13/vm/*" + zip -d ${REPO_ROOT}/build/$(basename $dir).zip "trem13/cgame.*" + zip -d ${REPO_ROOT}/build/$(basename $dir).zip "trem13/ui.*" + fi +done +chmod -R ugo+rw build diff --git a/src/client/cl_avi.cpp b/src/client/cl_avi.cpp index d533e604e..d4042f08a 100644 --- a/src/client/cl_avi.cpp +++ b/src/client/cl_avi.cpp @@ -608,7 +608,16 @@ bool CL_CloseAVI( void ) // Write index // Open the temp index file +#ifdef NEW_FILESYSTEM + indexSize = 0; + { char path[FS_MAX_PATH]; + if(fs_generate_path_writedir(FS_GetCurrentGameDir(), idxFileName, + 0, FS_ALLOW_DIRECTORIES, path, sizeof(path))) + afd.idxF = fs_direct_read_handle_open(0, path, (unsigned int *)&indexSize); } + if(indexSize <= 0) +#else if( ( indexSize = FS_FOpenFileRead( idxFileName, &afd.idxF, true ) ) <= 0 ) +#endif { FS_FCloseFile( afd.f ); return false; diff --git a/src/client/cl_cgame.cpp b/src/client/cl_cgame.cpp index f16dc35c5..3b62b1600 100644 --- a/src/client/cl_cgame.cpp +++ b/src/client/cl_cgame.cpp @@ -384,6 +384,9 @@ void CL_ShutdownCGame( void ) { return; } VM_Call( cls.cgame, CG_SHUTDOWN ); +#ifdef NEW_FILESYSTEM + fs_close_owner_handles(FS_HANDLEOWNER_CGAME); +#endif VM_Free( cls.cgame ); cls.cgame = NULL; } @@ -459,17 +462,33 @@ intptr_t CL_CgameSystemCalls( intptr_t *args ) Cmd_LiteralArgsBuffer( (char*)VMA(1), args[2] ); return 0; case CG_FS_FOPENFILE: +#ifdef NEW_FILESYSTEM + return FS_FOpenFileByModeOwner( (const char*)VMA(1), (fileHandle_t*)VMA(2), (FS_Mode)args[3], FS_HANDLEOWNER_CGAME ); +#else return FS_FOpenFileByMode( (const char*)VMA(1), (fileHandle_t*)VMA(2), (FS_Mode)args[3] ); +#endif case CG_FS_READ: +#ifdef NEW_FILESYSTEM + if(fs_handle_get_owner(args[3]) != FS_HANDLEOWNER_CGAME) return 0; +#endif FS_Read( VMA(1), args[2], args[3] ); return 0; case CG_FS_WRITE: +#ifdef NEW_FILESYSTEM + if(fs_handle_get_owner(args[3]) != FS_HANDLEOWNER_CGAME) return 0; +#endif FS_Write( VMA(1), args[2], args[3] ); return 0; case CG_FS_FCLOSEFILE: +#ifdef NEW_FILESYSTEM + if(fs_handle_get_owner(args[1]) != FS_HANDLEOWNER_CGAME) return 0; +#endif FS_FCloseFile( args[1] ); return 0; case CG_FS_SEEK: +#ifdef NEW_FILESYSTEM + if(fs_handle_get_owner(args[1]) != FS_HANDLEOWNER_CGAME) return 0; +#endif return FS_Seek( (fileHandle_t)args[1], args[2], (FS_Origin)args[3] ); case CG_FS_GETFILELIST: return FS_GetFileList( (const char*)VMA(1), (const char*)VMA(2), (char*)VMA(3), args[4] ); @@ -786,14 +805,22 @@ void CL_InitCGame( void ) { mapname = Info_ValueForKey( info, "mapname" ); Com_sprintf( cl.mapname, sizeof( cl.mapname ), "maps/%s.bsp", mapname ); +#ifdef NEW_FILESYSTEM + // Give the map pk3 slightly higher precedence in the filesystem to help + // load the correct textures and such + fs_register_current_map(cl.mapname); +#endif + // load the dll or bytecode interpret = (vmInterpret_t)Cvar_VariableValue("vm_cgame"); +#ifndef NEW_FILESYSTEM if(cl_connectedToPureServer) { // if sv_pure is set we only allow qvms to be loaded if(interpret != VMI_COMPILED && interpret != VMI_BYTECODE) interpret = VMI_COMPILED; } +#endif cls.cgame = VM_Create( "cgame", CL_CgameSystemCalls, interpret ); if ( !cls.cgame ) { diff --git a/src/client/cl_curl.cpp b/src/client/cl_curl.cpp index 0c81985f6..ebb296dc4 100644 --- a/src/client/cl_curl.cpp +++ b/src/client/cl_curl.cpp @@ -259,6 +259,10 @@ void CL_cURL_BeginDownload( const char *localName, const char *remoteURL ) CL_cURL_Cleanup(); Q_strncpyz(clc.downloadURL, remoteURL, sizeof(clc.downloadURL)); Q_strncpyz(clc.downloadName, localName, sizeof(clc.downloadName)); +#ifdef NEW_FILESYSTEM + if(!clc.activeCURLNotGameRelated) Com_sprintf(clc.downloadTempName, sizeof(clc.downloadTempName), "download.temp"); + else +#endif Com_sprintf(clc.downloadTempName, sizeof(clc.downloadTempName), "%s.tmp", localName); @@ -347,7 +351,14 @@ void CL_cURL_PerformDownload(void) return; } FS_FCloseFile(clc.download); +#ifdef NEW_FILESYSTEM + clc.download = 0; +#endif if(msg->msg == CURLMSG_DONE && msg->data.result == CURLE_OK) { +#ifdef NEW_FILESYSTEM + if(!clc.activeCURLNotGameRelated) fs_finalize_download(); + else +#endif FS_SV_Rename(clc.downloadTempName, clc.downloadName, false); clc.downloadRestart = true; } @@ -355,10 +366,19 @@ void CL_cURL_PerformDownload(void) long code; qcurl_easy_getinfo(msg->easy_handle, CURLINFO_RESPONSE_CODE, &code); +#ifdef NEW_FILESYSTEM + if(!clc.activeCURLNotGameRelated) Com_Printf("Download Error: %s Code: %ld URL: %s\n", + qcurl_easy_strerror(msg->data.result), code, clc.downloadURL); + else +#endif Com_Error(ERR_DROP, "Download Error: %s Code: %ld URL: %s", qcurl_easy_strerror(msg->data.result), code, clc.downloadURL); } +#ifdef NEW_FILESYSTEM + if(clc.activeCURLNotGameRelated) CL_cURL_Shutdown(); + else +#endif CL_NextDownload(); } diff --git a/src/client/cl_main.cpp b/src/client/cl_main.cpp index 417c2ef5e..822ac0ca9 100644 --- a/src/client/cl_main.cpp +++ b/src/client/cl_main.cpp @@ -186,6 +186,8 @@ static void CL_BrowseDemos_f() { FS_OpenBaseGamePath( "demos/" ); } static void CL_BrowseScreenShots_f() { FS_OpenBaseGamePath( "screenshots/" ); } +static void CL_BrowseBasemod_f() { FS_OpenModPath( "basemod" ); } + #ifdef USE_MUMBLE static void CL_UpdateMumble(void) { @@ -736,6 +738,9 @@ static char demoName[MAX_QPATH]; // compiler bug workaround static void CL_Record_f(void) { + char check_demo_name1[MAX_QPATH]; + char check_demo_name2[MAX_QPATH]; + char check_demo_name3[MAX_QPATH]; char name[MAX_OSPATH]; byte bufData[MAX_MSGLEN]; msg_t buf; @@ -774,7 +779,24 @@ static void CL_Record_f(void) if (Cmd_Argc() == 2) { const char *s = Cmd_Argv(1); + Q_strncpyz(demoName, s, sizeof(demoName)); + Com_sprintf( + check_demo_name1, sizeof(check_demo_name1), + "demos/%s.%s71", demoName, DEMOEXT); + Com_sprintf( + check_demo_name2, sizeof(check_demo_name2), + "demos/%s.%s69", demoName, DEMOEXT); + Com_sprintf( + check_demo_name3, sizeof(check_demo_name3), + "demos/%s.%s70", demoName, DEMOEXT); + if ( + FS_FileExistsInBaseGame(check_demo_name1) || + FS_FileExistsInBaseGame(check_demo_name2) || + FS_FileExistsInBaseGame(check_demo_name3)) { + Com_Printf(S_COLOR_RED "Demo name already exists, pick another name.\n" S_COLOR_WHITE); + return; + } Com_sprintf( name, sizeof(name), "demos/%s.%s%d", demoName, DEMOEXT, @@ -789,9 +811,29 @@ static void CL_Record_f(void) for (number = 0; number <= 9999; number++) { CL_DemoFilename(number, demoName, sizeof(demoName)); - Com_sprintf(name, sizeof(name), "demos/%s.%s%d", demoName, DEMOEXT, PROTOCOL_VERSION); - if (!FS_FileExists(name)) break; // file doesn't exist + Com_sprintf( + check_demo_name1, sizeof(check_demo_name1), + "demos/%s.%s71", demoName, DEMOEXT); + Com_sprintf( + check_demo_name2, sizeof(check_demo_name2), + "demos/%s.%s69", demoName, DEMOEXT); + Com_sprintf( + check_demo_name3, sizeof(check_demo_name3), + "demos/%s.%s70", demoName, DEMOEXT); + + if ( + !FS_FileExistsInBaseGame(check_demo_name1) && + !FS_FileExistsInBaseGame(check_demo_name2) && + !FS_FileExistsInBaseGame(check_demo_name3)) { + // file doesn't exist, so use this name + Com_sprintf( + name, sizeof(name), + "demos/%s.%s%d", demoName, DEMOEXT, + (clc.netchan.alternateProtocol == 0 ? + PROTOCOL_VERSION : clc.netchan.alternateProtocol == 1 ? 70 : 69)); + break; + } } } @@ -970,6 +1012,13 @@ void CL_DemoCompleted(void) } } +#ifdef NEW_FILESYSTEM + if(!*Cvar_VariableString("nextdemo")) + { + // Do a full disconnect to ensure new UI gets loaded + CL_Disconnect_f(); + } +#endif CL_Disconnect(true); CL_NextDemo(); } @@ -1111,6 +1160,11 @@ void CL_PlayDemo_f(void) return; } +#ifdef NEW_FILESYSTEM + // Refresh here in case a demo was just recorded or manually added + fs_refresh_auto(); +#endif + // make sure a local server is killed // 2 means don't force disconnect of local client Cvar_Set("sv_killserver", "2"); @@ -1413,9 +1467,15 @@ void CL_Disconnect(bool showMainMenu) CL_WritePacket(); } +#ifdef NEW_FILESYSTEM + fs_disconnect_cleanup(); + if(!clc.cURLReconnecting) fs_clear_attempted_downloads(); + clc.cURLReconnecting = qfalse; +#else // Remove pure paks FS_PureServerSetLoadedPaks("", ""); FS_PureServerSetReferencedPaks("", ""); +#endif CL_ClearState(); @@ -1429,8 +1489,10 @@ void CL_Disconnect(bool showMainMenu) // allow cheats locally Cvar_Set("sv_cheats", "1"); +#ifndef NEW_FILESYSTEM // not connected to a pure server anymore cl_connectedToPureServer = false; +#endif #ifdef USE_VOIP // not connected to voip server anymore. @@ -1819,6 +1881,7 @@ static void CL_Snd_Shutdown(void) S_Shutdown(); cls.soundStarted = false; } +#ifndef NEW_FILESYSTEM /* ================== CL_PK3List_f @@ -1832,6 +1895,7 @@ CL_PureList_f ================== */ static void CL_ReferencedPK3List_f(void) { Com_Printf("Referenced PK3 Names: %s\n", FS_ReferencedPakNames(false)); } +#endif /* ================== @@ -1895,23 +1959,32 @@ static void CL_DownloadsComplete(void) CL_cURL_Shutdown(); if (clc.cURLDisconnected) { +#ifdef NEW_FILESYSTEM + clc.downloadRestart = qfalse; + clc.cURLReconnecting = qtrue; // Don't reset attempted downloads +#else if (clc.downloadRestart) { if (!clc.activeCURLNotGameRelated) FS_Restart(clc.checksumFeed); clc.downloadRestart = false; } +#endif clc.cURLDisconnected = false; if (!clc.activeCURLNotGameRelated) CL_Reconnect_f(); return; } } +#ifndef NEW_FILESYSTEM // if we downloaded files we need to restart the file system +#endif if (clc.downloadRestart) { clc.downloadRestart = false; +#ifndef NEW_FILESYSTEM FS_Restart(clc.checksumFeed); // We possibly downloaded a pak, restart the file system to load it +#endif // inform the server so we get new gamestate info CL_AddReliableCommand("donedl", false); @@ -1945,7 +2018,11 @@ static void CL_DownloadsComplete(void) // this will also (re)load the UI // if this is a local client then only the client part of the hunk // will be cleared, note that this is done after the hunk mark has been set +#ifdef NEW_FILESYSTEM + if(!FS_ConditionalRestart(clc.checksumFeed, false)) CL_FlushMemory(); +#else CL_FlushMemory(); +#endif // initialize the CGame cls.cgameStarted = true; @@ -1977,7 +2054,11 @@ static void CL_BeginDownload(const char *localName, const char *remoteName) localName, remoteName); Q_strncpyz(clc.downloadName, localName, sizeof(clc.downloadName)); +#ifdef NEW_FILESYSTEM + Com_sprintf(clc.downloadTempName, sizeof(clc.downloadTempName), "download.temp"); +#else Com_sprintf(clc.downloadTempName, sizeof(clc.downloadTempName), "%s.tmp", localName); +#endif // Set so UI gets access to it Cvar_Set("cl_downloadName", remoteName); @@ -2003,6 +2084,86 @@ A download completed or failed */ void CL_NextDownload(void) { +#ifdef NEW_FILESYSTEM + // Attempts to initiate a download, or calls CL_DownloadsComplete if no more downloads are available + *clc.downloadTempName = *clc.downloadName = 0; + Cvar_Set("cl_downloadName", ""); + + while(1) { + char *remoteName, *localName; + bool curl_already_attempted = false; + + // Update remaining download stats for UI + clc.downloadDone = clc.downloadTotal - fs_estimate_remaining_downloads(); + Cvar_SetValue("cl_downloadDone", clc.downloadDone); + + // Get next potential download + fs_advance_next_needed_download(clc.cURLDisconnected); + if(!fs_get_current_download_info(&localName, &remoteName, &curl_already_attempted)) { + CL_DownloadsComplete(); + return; } + + // Check some skip conditions + if(!(cl_allowDownload->integer & DLF_ENABLE)) { + Com_Printf("WARNING: Skipping download '%s' because all downloads are disabled " + "on your client (cl_allowDownload is %d)\n", localName, cl_allowDownload->integer); + fs_advance_download(); + continue; } + if(!Q_stricmp(clc.servername, "localhost")) { + // Don't do any downloads when connected to a local game + // Otherwise the game could try to download from itself under certain circumstances + Com_Printf("WARNING: Skipping download '%s' because the game appears to be local.\n", localName); + fs_advance_download(); + continue; } + + // Attempt cURL download + if(curl_already_attempted) { + Com_Printf("NOTE: Attempting UDP download for '%s' because a cURL download appears" + " to have been unsuccessful.\n", localName); } + + else if(!(cl_allowDownload->integer & DLF_NO_REDIRECT)) { + // cURL download enabled in client + if(!*clc.sv_dlURL) { + Com_Printf("NOTE: cURL download not available because server did not set sv_dlURL\n"); } + else if(clc.sv_allowDownload & DLF_NO_REDIRECT) { + Com_Printf("NOTE: cURL download not available due to server setting" + " (sv_allowDownload is %d)\n", clc.sv_allowDownload); } + else if(!CL_cURL_Init()) { + Com_Printf("NOTE: could not load cURL library\n"); } + else { + // Begin cURL Download + fs_register_current_download_attempt(qtrue); + // Using remoteName instead of localName for UI purposes + CL_cURL_BeginDownload(remoteName, va("%s/%s", clc.sv_dlURL, remoteName)); + if(!clc.cURLDisconnected) clc.downloadRestart = qtrue; + return; } } + + else if(!(clc.sv_allowDownload & DLF_NO_REDIRECT) && *clc.sv_dlURL) { + // cURL download not enabled in client, but enabled on server + Com_Printf("NOTE: cURL download not available due to client setting" + " (cl_allowDownload is %d)\n", cl_allowDownload->integer); } + + // Attempt UDP download + if((cl_allowDownload->integer & DLF_NO_UDP)) { + Com_Printf("WARNING: Skipping download '%s' because UDP downloads are disabled" + " on your client (cl_allowDownload is %d)\n", + localName, cl_allowDownload->integer); + fs_advance_download(); + continue; } + else if(!(clc.sv_allowDownload & DLF_ENABLE) || (clc.sv_allowDownload & DLF_NO_UDP)) { + Com_Printf("WARNING: Skipping download '%s' because UDP downloads appear to be disabled" + " on the server (sv_allowDownload is %d)\n", + localName, clc.sv_allowDownload); + fs_advance_download(); + continue; } + else { + // Begin UDP Download + Com_Printf("Starting UDP download for '%s'\n", localName); + fs_register_current_download_attempt(qfalse); + CL_BeginDownload( localName, remoteName ); + clc.downloadRestart = qtrue; + return; } } +#else char *s; char *remoteName, *localName; bool useCURL = false; @@ -2220,6 +2381,7 @@ void CL_NextDownload(void) } CL_DownloadsComplete(); +#endif } /* @@ -2232,6 +2394,27 @@ and determine if we need to download them */ void CL_InitDownloads(void) { +#ifdef NEW_FILESYSTEM + clc.state = CA_CONNECTED; + + *clc.downloadTempName = *clc.downloadName = 0; + Cvar_Set("cl_downloadName", ""); + if (clc.download) + { + FS_FCloseFile(clc.download); + clc.download = 0; + } + + fs_print_download_list(); + + // Initialize remaining download stats for UI + clc.downloadDone = 0; + clc.downloadTotal = fs_estimate_remaining_downloads(); + Cvar_Set("cl_downloadDone", "0"); + Cvar_SetValue("cl_downloadTotal", clc.downloadTotal); + + CL_NextDownload(); +#else char *s; if (FS_ComparePaks(clc.downloadList, sizeof(clc.downloadList), true)) @@ -2260,6 +2443,7 @@ void CL_InitDownloads(void) } } CL_DownloadsComplete(); +#endif } /* @@ -3212,6 +3396,9 @@ Also called by Com_Error */ void CL_FlushMemory(void) { +#ifdef NEW_FILESYSTEM + fs_advance_cache_stage(); +#endif CL_ClearMemory(false); CL_StartHunkUsers(false); } @@ -3261,8 +3448,15 @@ static void CL_Vid_Restart_f(void) CL_ShutdownRef(); // client is no longer pure untill new checksums are sent CL_ResetPureClientAtServer(); +#ifdef NEW_FILESYSTEM + // Refresh in case files have been manually changed + fs_refresh_auto(); + + // New filesystem currently doesn't store ui and cgame references +#else // clear pak references FS_ClearPakReferences(FS_UI_REF | FS_CGAME_REF); +#endif // reinitialize the filesystem if the game directory or checksum has changed cls.rendererStarted = false; @@ -3536,7 +3730,11 @@ static void CL_SetServerInfo(serverInfo_t *server, const char *info, int ping) server->minPing = atoi(Info_ValueForKey(info, "minping")); server->maxPing = atoi(Info_ValueForKey(info, "maxping")); const char *game = Info_ValueForKey(info, "game"); +#ifdef NEW_FILESYSTEM + Q_strncpyz(server->game, (game[0]) ? game : "base", MAX_NAME_LENGTH); +#else Q_strncpyz(server->game, (game[0]) ? game : BASEGAME, MAX_NAME_LENGTH); +#endif } server->ping = ping; } @@ -4762,7 +4960,13 @@ static void CL_InitRef(void) ri.FS_WriteFile = FS_WriteFile; ri.FS_FreeFileList = FS_FreeFileList; ri.FS_ListFiles = FS_ListFiles; +#ifdef NEW_FILESYSTEM + // This function is not implemented in the new filesystem, and it does + // not appear to actually be used in the renderer. + ri.FS_FileIsInPAK = 0; +#else ri.FS_FileIsInPAK = FS_FileIsInPAK; +#endif ri.FS_FileExists = FS_FileExists; ri.Cvar_Get = Cvar_Get; ri.Cvar_Set = Cvar_Set; @@ -4787,6 +4991,15 @@ static void CL_InitRef(void) ri.Sys_GLimpInit = Sys_GLimpInit; ri.Sys_LowPhysicalMemory = Sys_LowPhysicalMemory; +#ifdef NEW_FILESYSTEM + ri.fs_general_lookup = fs_general_lookup; + ri.fs_image_lookup = fs_image_lookup; + ri.fs_shader_lookup = fs_shader_lookup; + ri.fs_read_shader = fs_read_shader; + ri.fs_file_extension = fs_file_extension; + ri.fs_files_from_same_pk3 = fs_files_from_same_pk3; +#endif + ret = GetRefAPI(REF_API_VERSION, &ri); #if defined __USEA3D && defined __A3D_GEOM @@ -5007,8 +5220,10 @@ void CL_Init(void) Cmd_AddCommand("ping", CL_Ping_f); Cmd_AddCommand("serverstatus", CL_ServerStatus_f); Cmd_AddCommand("showip", CL_ShowIP_f); +#ifndef NEW_FILESYSTEM Cmd_AddCommand("fs_openedList", CL_OpenedPK3List_f); Cmd_AddCommand("fs_referencedList", CL_ReferencedPK3List_f); +#endif Cmd_AddCommand("model", CL_SetModel_f); Cmd_AddCommand("video", CL_Video_f); Cmd_AddCommand("stopvideo", CL_StopVideo_f); @@ -5018,6 +5233,7 @@ void CL_Init(void) Cmd_AddCommand("browseHomepath", CL_BrowseHomepath_f); Cmd_AddCommand("browseDemos", CL_BrowseDemos_f); Cmd_AddCommand("browseScreenShots", CL_BrowseScreenShots_f); + Cmd_AddCommand("browseBasemod", CL_BrowseBasemod_f); CL_InitRef(); diff --git a/src/client/cl_parse.cpp b/src/client/cl_parse.cpp index 111211a61..c18d3fc2e 100644 --- a/src/client/cl_parse.cpp +++ b/src/client/cl_parse.cpp @@ -342,7 +342,9 @@ void CL_ParseSnapshot( msg_t *msg ) { //===================================================================== +#ifndef NEW_FILESYSTEM bool cl_connectedToPureServer; +#endif bool cl_connectedToCheatServer; /* @@ -389,7 +391,11 @@ void CL_SystemInfoChanged( void ) { s = Info_ValueForKey( systemInfo, "sv_referencedPaks" ); t = Info_ValueForKey( systemInfo, "sv_referencedPakNames" ); +#ifdef NEW_FILESYSTEM + fs_register_download_list(s, t); +#else FS_PureServerSetReferencedPaks( s, t ); +#endif gameSet = false; // scan through all the variables in the systeminfo and locally set cvars to match @@ -405,17 +411,26 @@ void CL_SystemInfoChanged( void ) { // ehw! if (!Q_stricmp(key, "fs_game")) { +#ifndef NEW_FILESYSTEM if(FS_CheckDirTraversal(value)) { Com_Printf(S_COLOR_YELLOW "WARNING: Server sent invalid fs_game value %s\n", value); continue; } +#endif gameSet = true; } else if ( clc.demoplaying && Q_stricmp(key, "sv_pure" ) ) { continue; } +#ifdef NEW_FILESYSTEM + if (!Q_stricmp(key, "sv_pure")) + { + fs_set_connected_server_sv_pure_value(atoi(value)); + } +#endif + if((cvar_flags = Cvar_Flags(key)) == CVAR_NONEXISTENT) Cvar_Get(key, value, CVAR_SERVER_CREATED | CVAR_ROM); else @@ -438,7 +453,11 @@ void CL_SystemInfoChanged( void ) { if ( !gameSet && *Cvar_VariableString("fs_game") ) { Cvar_Set( "fs_game", "" ); } +#ifdef NEW_FILESYSTEM + fs_set_connected_server_profile(clc.netchan.alternateProtocol); +#else cl_connectedToPureServer = (bool)Cvar_VariableValue( "sv_pure" ); +#endif } /* @@ -546,7 +565,9 @@ void CL_ParseGamestate( msg_t *msg ) { Q_strncpyz(cl_oldGame, oldGame, sizeof(cl_oldGame)); } +#ifndef NEW_FILESYSTEM FS_ConditionalRestart(clc.checksumFeed, false); +#endif // This used to call CL_StartHunkUsers, but now we enter the download state before loading the // cgame @@ -639,7 +660,11 @@ void CL_ParseDownload ( msg_t *msg ) { clc.download = 0; // rename the file +#ifdef NEW_FILESYSTEM + fs_finalize_download(); +#else FS_SV_Rename( clc.downloadTempName, clc.downloadName, false ); +#endif } // send intentions now diff --git a/src/client/cl_rest.cpp b/src/client/cl_rest.cpp index c1ff2fcf1..aff9d6e60 100644 --- a/src/client/cl_rest.cpp +++ b/src/client/cl_rest.cpp @@ -36,11 +36,16 @@ bool MakeDir(std::string destdir, std::string basegame) destpath += basegame; } +#ifdef NEW_FILESYSTEM + char buffer[FS_MAX_PATH]; + fs_generate_path(destpath.c_str(), 0, 0, FS_NO_SANITIZE|FS_CREATE_DIRECTORIES, 0, 0, buffer, sizeof(buffer)); +#else if ( *destpath.rbegin() != '/' ) destpath += '/'; // XXX FS_CreatePath requires a trailing slash. // Maybe the assumption is that a file listing might be included? FS_CreatePath(destpath.c_str()); +#endif return true; } diff --git a/src/client/cl_ui.cpp b/src/client/cl_ui.cpp index 4052fe6b0..f56e90f5d 100644 --- a/src/client/cl_ui.cpp +++ b/src/client/cl_ui.cpp @@ -52,7 +52,11 @@ void LAN_LoadCachedServers(void) fileHandle_t fileIn; cls.numglobalservers = cls.numfavoriteservers = 0; cls.numGlobalServerAddresses = 0; +#ifdef NEW_FILESYSTEM + if (FS_SV_FOpenFileRead("servercache.dat", &fileIn) > 0) +#else if (FS_SV_FOpenFileRead("servercache.dat", &fileIn)) +#endif { FS_Read(&cls.numglobalservers, sizeof(int), fileIn); FS_Read(&cls.numfavoriteservers, sizeof(int), fileIn); @@ -888,17 +892,30 @@ intptr_t CL_UISystemCalls(intptr_t *args) return 0; case UI_FS_FOPENFILE: +#ifdef NEW_FILESYSTEM + return FS_FOpenFileByModeOwner((const char *)VMA(1), (fileHandle_t *)VMA(2), (FS_Mode)args[3], FS_HANDLEOWNER_UI); +#else return FS_FOpenFileByMode((const char *)VMA(1), (fileHandle_t *)VMA(2), (FS_Mode)args[3]); +#endif case UI_FS_READ: +#ifdef NEW_FILESYSTEM + if(fs_handle_get_owner(args[3]) != FS_HANDLEOWNER_UI) return 0; +#endif FS_Read(VMA(1), args[2], args[3]); return 0; case UI_FS_WRITE: +#ifdef NEW_FILESYSTEM + if(fs_handle_get_owner(args[3]) != FS_HANDLEOWNER_UI) return 0; +#endif FS_Write(VMA(1), args[2], args[3]); return 0; case UI_FS_FCLOSEFILE: +#ifdef NEW_FILESYSTEM + if(fs_handle_get_owner(args[1]) != FS_HANDLEOWNER_UI) return 0; +#endif FS_FCloseFile(args[1]); return 0; @@ -906,6 +923,9 @@ intptr_t CL_UISystemCalls(intptr_t *args) return FS_GetFileList((const char *)VMA(1), (const char *)VMA(2), (char *)VMA(3), args[4]); case UI_FS_SEEK: +#ifdef NEW_FILESYSTEM + if(fs_handle_get_owner(args[1]) != FS_HANDLEOWNER_UI) return 0; +#endif return FS_Seek((fileHandle_t)args[1], args[2], (FS_Origin)args[3]); case UI_R_REGISTERMODEL: @@ -1187,6 +1207,9 @@ void CL_ShutdownUI(void) return; } VM_Call(cls.ui, UI_SHUTDOWN); +#ifdef NEW_FILESYSTEM + fs_close_owner_handles(FS_HANDLEOWNER_UI); +#endif VM_Free(cls.ui); cls.ui = NULL; } @@ -1200,11 +1223,13 @@ void CL_InitUI(void) { // load the dll or bytecode vmInterpret_t interpret = (vmInterpret_t)Cvar_VariableValue("vm_ui"); +#ifndef NEW_FILESYSTEM if (cl_connectedToPureServer) { // if sv_pure is set we only allow qvms to be loaded if (interpret != VMI_COMPILED && interpret != VMI_BYTECODE) interpret = VMI_COMPILED; } +#endif cls.ui = VM_Create("ui", CL_UISystemCalls, interpret); if (!cls.ui) diff --git a/src/client/client.h b/src/client/client.h index ba99d0826..662e05146 100644 --- a/src/client/client.h +++ b/src/client/client.h @@ -276,6 +276,9 @@ struct clientConnection_t { bool cURLEnabled; bool cURLUsed; bool cURLDisconnected; +#ifdef NEW_FILESYSTEM + bool cURLReconnecting; +#endif char downloadURL[MAX_OSPATH]; CURL *downloadCURL; @@ -290,7 +293,9 @@ struct clientConnection_t { int downloadBlock; // block we are waiting for int downloadCount; // how many bytes we got int downloadSize; // how many bytes we got +#ifndef NEW_FILESYSTEM char downloadList[MAX_INFO_STRING]; // list of paks we need to download +#endif bool downloadRestart; // if true, we need to do another FS_Restart because we downloaded a pak char newsString[MAX_NEWS_STRING]; @@ -595,7 +600,9 @@ const char *Key_KeynumToString(int keynum); // // cl_parse.c // +#ifndef NEW_FILESYSTEM extern bool cl_connectedToPureServer; +#endif extern bool cl_connectedToCheatServer; #ifdef USE_VOIP diff --git a/src/client/snd_codec.cpp b/src/client/snd_codec.cpp index a794b90b3..4ea90c128 100644 --- a/src/client/snd_codec.cpp +++ b/src/client/snd_codec.cpp @@ -38,6 +38,33 @@ then tries all supported codecs. */ static void *S_CodecGetSound(const char *filename, snd_info_t *info) { +#ifdef NEW_FILESYSTEM + char localName[MAX_QPATH]; + const fsc_file_t *file; + const char *extension; + snd_codec_t *codec; + + COM_StripExtension(filename, localName, MAX_QPATH); + + // Look up the file + file = fs_sound_lookup(localName, 0, false); + if(!file) { + Com_Printf(S_COLOR_YELLOW "WARNING: Failed to %s sound %s!\n", info ? "load" : "open", filename); + return NULL; } + + // Get extension + extension = fs_file_extension(file); + if(!extension) Com_Error(ERR_DROP, "S_CodecGetSound got file with invalid extension from fs_sound_lookup"); + + for(codec=codecs; codec; codec=codec->next) { + if(!Q_stricmp(extension, codec->ext)) { + // Load + if(info) return codec->load(va("%s.%s", localName, codec->ext), info); + else return codec->open(va("%s.%s", localName, codec->ext)); } } + + Com_Error(ERR_DROP, "S_CodecGetSound got file with unknown extension from fs_sound_lookup"); + return NULL; +#else snd_codec_t *codec; snd_codec_t *orgCodec = NULL; bool orgNameFailed = false; @@ -115,6 +142,7 @@ static void *S_CodecGetSound(const char *filename, snd_info_t *info) Com_Printf(S_COLOR_YELLOW "WARNING: Failed to %s sound %s!\n", info ? "load" : "open", filename); return NULL; +#endif } /* diff --git a/src/filesystem/fs_commands.cpp b/src/filesystem/fs_commands.cpp new file mode 100644 index 000000000..ddc47cdd8 --- /dev/null +++ b/src/filesystem/fs_commands.cpp @@ -0,0 +1,274 @@ +/* +=========================================================================== +Copyright (C) 1999-2005 Id Software, Inc. +Copyright (C) 2017 Noah Metzger (chomenor@gmail.com) + +This file is part of Quake III Arena source code. + +Quake III Arena source code is free software; you can redistribute it +and/or modify it under the terms of the GNU General Public License as +published by the Free Software Foundation; either version 2 of the License, +or (at your option) any later version. + +Quake III Arena source code is distributed in the hope that it will be +useful, but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with Quake III Arena source code; if not, write to the Free Software +Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA +=========================================================================== +*/ + +#ifdef NEW_FILESYSTEM +#include "fslocal.h" + +/* ******************************************************************************** */ +// Lookup Test Commands +/* ******************************************************************************** */ + +static void cmd_find_file(void) +{ + if (Cmd_Argc() < 2) + { + Com_Printf("Usage: find_file \n"); + return; + } + + fs_general_lookup(Cmd_Argv(1), atoi(Cmd_Argv(2)), qtrue); +} + +static void cmd_find_shader(void) +{ + if (Cmd_Argc() < 2) + { + Com_Printf("Usage: find_shader \n"); + return; + } + + fs_shader_lookup(Cmd_Argv(1), atoi(Cmd_Argv(2)), qtrue); +} + +static void cmd_find_sound(void) +{ + if (Cmd_Argc() < 2) + { + Com_Printf("Usage: find_sound \n"); + return; + } + + fs_sound_lookup(Cmd_Argv(1), atoi(Cmd_Argv(2)), qtrue); +} + +static void cmd_find_vm(void) +{ + if (Cmd_Argc() != 2) + { + Com_Printf("Usage: find_vm \n"); + return; + } + + fs_vm_lookup(Cmd_Argv(1), qfalse, qtrue, 0); +} + +static void cmd_compare(void) +{ + if (Cmd_Argc() != 3) + { + Com_Printf( + "Usage: compare \n\nRun this command following a 'find_file', 'find_shader', " + "'find_sound', or 'find_vm' command and specify the resource numbers you wish to compare.\n"); + return; + } + + debug_resource_comparison(atoi(Cmd_Argv(1)), atoi(Cmd_Argv(2))); +} + +/* ******************************************************************************** */ +// Other Commands +/* ******************************************************************************** */ + +static void cmd_fs_refresh(void) +{ + // Runs filesystem refresh + // Can be called with parameter 1 for quiet mode + fs_refresh_auto_ext(qtrue, atoi(Cmd_Argv(1)) ? qtrue : qfalse); +} + +static void cmd_readcache_debug(void) { fs_readcache_debug(); } + +static void cmd_indexcache_write(void) { fs_indexcache_write(); } + +static void FS_Dir_f(void) +{ + const char *path; + const char *extension; + char **dirnames; + int ndirs; + int i; + + if (Cmd_Argc() < 2 || Cmd_Argc() > 3) + { + Com_Printf("usage: dir [extension]\n"); + return; + } + + if (Cmd_Argc() == 2) + { + path = Cmd_Argv(1); + extension = ""; + } + else + { + path = Cmd_Argv(1); + extension = Cmd_Argv(2); + } + + Com_Printf("Directory of %s %s\n", path, extension); + Com_Printf("---------------\n"); + + dirnames = FS_ListFiles(path, extension, &ndirs); + + for (i = 0; i < ndirs; i++) + { + Com_Printf("%s\n", dirnames[i]); + } + FS_FreeFileList(dirnames); +} + +static void FS_NewDir_f(void) +{ + const char *filter; + char **dirnames; + int ndirs; + int i; + + if (Cmd_Argc() < 2) + { + Com_Printf("usage: fdir \n"); + Com_Printf("example: fdir *q3dm*.bsp\n"); + return; + } + + filter = Cmd_Argv(1); + + Com_Printf("---------------\n"); + + dirnames = FS_FlagListFilteredFiles("", "", filter, &ndirs, 0); + + for (i = 0; i < ndirs; i++) + { + Com_Printf("%s\n", dirnames[i]); + } + Com_Printf("%d files listed\n", ndirs); + FS_FreeFileList(dirnames); +} + +static void FS_Which_f(void) +{ + // The lookup functions are more powerful, but this is kept for + // users who are familiar with it + const char *filename = Cmd_Argv(1); + const fsc_file_t *file; + + if (!filename[0]) + { + Com_Printf("Usage: which \n"); + return; + } + + // qpaths are not supposed to have a leading slash + if (filename[0] == '/' || filename[0] == '\\') + { + filename++; + } + + file = fs_general_lookup(filename, 0, qfalse); + if (file) + fs_print_file_location(file); + else + Com_Printf("File not found: \"%s\"\n", filename); +} + +static void FS_TouchFile_f(void) +{ + fileHandle_t f; + + if (Cmd_Argc() != 2) + { + Com_Printf("Usage: touchFile \n"); + return; + } + + FS_FOpenFileRead(Cmd_Argv(1), &f, qfalse); + if (f) + { + FS_FCloseFile(f); + } +} + +static void FS_Path_f(void) +{ + // Quick implementation without sorting + fsc_hashtable_iterator_t hti; + fsc_pk3_hash_map_entry_t *hash_entry; + int sourceid; + int i; + + for (sourceid = 0; sourceid < FS_MAX_SOURCEDIRS; ++sourceid) + { + if (!fs_sourcedirs[sourceid].active) + continue; + Com_Printf("Looking in %s (%s)\n", fs_sourcedirs[sourceid].name, fs_sourcedirs[sourceid].path); + + for (i = 0; i < fs.pk3_hash_lookup.bucket_count; ++i) + { + fsc_hashtable_open(&fs.pk3_hash_lookup, i, &hti); + while ((hash_entry = (fsc_pk3_hash_map_entry_t *)STACKPTRN(fsc_hashtable_next(&hti)))) + { + const fsc_file_direct_t *pak = (const fsc_file_direct_t *)STACKPTR(hash_entry->pk3); + if (pak->source_dir_id == sourceid && + !fs_file_disabled((fsc_file_t *)pak, FD_CHECK_FILE_ENABLED | FD_CHECK_READ_INACTIVE_MODS)) + { + char buffer[FS_FILE_BUFFER_SIZE]; + fs_file_to_buffer((fsc_file_t *)pak, buffer, sizeof(buffer), qfalse, qtrue, qfalse, qfalse); + Com_Printf("%s (%i files)\n", buffer, pak->pk3_subfile_count); + Com_Printf( + " hash(%i) core_pk3_position(%i)\n", (int)pak->pk3_hash, core_pk3_position(pak->pk3_hash)); + if (fs_connected_server_pure_state()) + Com_Printf(" %son the pure list\n", + pk3_list_lookup(&connected_server_pure_list, pak->pk3_hash) ? "" : "not "); + } + } + } + } + + Com_Printf("\n"); + fs_print_handle_list(); +} + +/* ******************************************************************************** */ +// Command Register Function +/* ******************************************************************************** */ + +void fs_register_commands(void) +{ + Cmd_AddCommand("find_file", cmd_find_file); + Cmd_AddCommand("find_shader", cmd_find_shader); + Cmd_AddCommand("find_sound", cmd_find_sound); + Cmd_AddCommand("find_vm", cmd_find_vm); + Cmd_AddCommand("compare", cmd_compare); + + Cmd_AddCommand("fs_refresh", cmd_fs_refresh); + Cmd_AddCommand("readcache_debug", cmd_readcache_debug); + Cmd_AddCommand("indexcache_write", cmd_indexcache_write); + + Cmd_AddCommand("dir", FS_Dir_f); + Cmd_AddCommand("fdir", FS_NewDir_f); + Cmd_AddCommand("which", FS_Which_f); + Cmd_AddCommand("touchfile", FS_TouchFile_f); + Cmd_AddCommand("path", FS_Path_f); +} + +#endif // NEW_FILESYSTEM diff --git a/src/filesystem/fs_download.cpp b/src/filesystem/fs_download.cpp new file mode 100644 index 000000000..b00650f76 --- /dev/null +++ b/src/filesystem/fs_download.cpp @@ -0,0 +1,499 @@ +/* +=========================================================================== +Copyright (C) 1999-2005 Id Software, Inc. +Copyright (C) 2017 Noah Metzger (chomenor@gmail.com) + +This file is part of Quake III Arena source code. + +Quake III Arena source code is free software; you can redistribute it +and/or modify it under the terms of the GNU General Public License as +published by the Free Software Foundation; either version 2 of the License, +or (at your option) any later version. + +Quake III Arena source code is distributed in the hope that it will be +useful, but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with Quake III Arena source code; if not, write to the Free Software +Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA +=========================================================================== +*/ + +#ifdef NEW_FILESYSTEM +#include "fslocal.h" + +#define MAX_DOWNLOAD_NAME 64 // Max length of the pk3 filename + +typedef struct download_entry_s { + struct download_entry_s *next; + unsigned int hash; + char *local_name; + char *remote_name; + char *filename; + char *mod_dir; +} download_entry_t; + +/* ******************************************************************************** */ +// Download List Handling +/* ******************************************************************************** */ + +static download_entry_t *current_download; +static download_entry_t *next_download; + +static void fs_free_download_entry(download_entry_t *entry) +{ + Z_Free(entry->local_name); + Z_Free(entry->remote_name); + Z_Free(entry->filename); + Z_Free(entry->mod_dir); + Z_Free(entry); +} + +void fs_advance_download(void) +{ + // Pops a download entry from next_download into current_download + if (current_download) + fs_free_download_entry(current_download); + current_download = next_download; + if (next_download) + next_download = next_download->next; +} + +static void fs_add_next_download(download_entry_t *entry) +{ + // Push a download entry into next_download + entry->next = next_download; + next_download = entry; +} + +static void fs_free_download_list(void) +{ + // Free all downloads in list + while (current_download || next_download) + fs_advance_download(); +} + +/* ******************************************************************************** */ +// Attempted Download Tracking +/* ******************************************************************************** */ + +// This section is used to prevent trying to unsuccessfully download the same file over +// and over again in the same session. + +pk3_list_t attempted_downloads_http; +pk3_list_t attempted_downloads; + +static void register_attempted_download(unsigned int hash, qboolean http) +{ + pk3_list_t *target = http ? &attempted_downloads_http : &attempted_downloads; + if (!target->ht.bucket_count) + pk3_list_initialize(target, 20); + pk3_list_insert(target, hash); +} + +static qboolean check_attempted_download(unsigned int hash, qboolean http) +{ + // Returns qtrue if download already attempted + pk3_list_t *target = http ? &attempted_downloads_http : &attempted_downloads; + return pk3_list_lookup(target, hash) ? qtrue : qfalse; +} + +void fs_register_current_download_attempt(qboolean http) +{ + FSC_ASSERT(current_download); + register_attempted_download(current_download->hash, http); +} + +void fs_clear_attempted_downloads(void) +{ + pk3_list_free(&attempted_downloads_http); + pk3_list_free(&attempted_downloads); +} + +/* ******************************************************************************** */ +// Needed Download Checks +/* ******************************************************************************** */ + +static bool dl_is_base_directory(const char *mod_dir) +{ + // Returns true if specified mod dir is considered a base directory for download purposes + if (!Q_stricmp(mod_dir, BASEGAME_1_1)) + return true; + if (!Q_stricmp(mod_dir, BASEGAME_GPP)) + return true; + if (!Q_stricmp(mod_dir, BASEGAME_1_3)) + return true; + return false; +} + +static qboolean entry_match_in_index(download_entry_t *entry, fsc_file_direct_t **different_moddir_match_out) +{ + // Returns qtrue if download entry matches a file already in main index + fsc_hashtable_iterator_t hti; + fsc_pk3_hash_map_entry_t *hashmap_entry; + bool core_pak = core_pk3_position(entry->hash) ? true : false; + bool target_is_base_directory = dl_is_base_directory(entry->mod_dir); + + fsc_hashtable_open(&fs.pk3_hash_lookup, entry->hash, &hti); + while ((hashmap_entry = (fsc_pk3_hash_map_entry_t *)STACKPTRN(fsc_hashtable_next(&hti)))) + { + fsc_file_direct_t *pk3 = (fsc_file_direct_t *)STACKPTR(hashmap_entry->pk3); + if (!fsc_is_file_enabled((fsc_file_t *)pk3, &fs)) + continue; + if (pk3->pk3_hash != entry->hash) + continue; + + if (core_pak && target_is_base_directory && dl_is_base_directory(fsc_get_mod_dir((fsc_file_t *)pk3, &fs))) + { + // Don't redownload core pk3s to different base directories regardless of fs_redownload_across_mods setting + return qtrue; + } + + if (fs_redownload_across_mods->integer && Q_stricmp(fsc_get_mod_dir((fsc_file_t *)pk3, &fs), entry->mod_dir)) + { + // If "fs_redownload_across_mods" is set, ignore match from different mod dir, + // but record it so fs_is_valid_download can display a warning later + if (different_moddir_match_out) + *different_moddir_match_out = pk3; + continue; + } + return qtrue; + } + + return qfalse; +} + +static qboolean fs_is_download_id_pak(download_entry_t *entry) +{ + char test_path[FS_MAX_PATH]; + Com_sprintf(test_path, sizeof(test_path), "%s/%s", entry->mod_dir, entry->filename); +#ifndef STANDALONE + if (FS_idPak(test_path, BASEGAME, FS_NODOWNLOAD_PAKS)) + return qtrue; + if (FS_idPak(test_path, BASETA, FS_NODOWNLOAD_PAKS_TEAMARENA)) + return qtrue; +#endif + return qfalse; +} + +static qboolean fs_is_valid_download(download_entry_t *entry, unsigned int recheck_hash, qboolean curl_disconnected) +{ + // Returns qtrue if file should be downloaded, qfalse otherwise. + // recheck_hash can be set to retest a file that was downloaded and has an unexpected hash + unsigned int hash = recheck_hash ? recheck_hash : entry->hash; + fsc_file_direct_t *different_moddir_match = 0; + + if (fs_read_only) + { + Com_Printf("WARNING: Ignoring download %s because filesystem is in read-only state.\n", entry->local_name); + return qfalse; + } + if (!Q_stricmp(entry->mod_dir, "basemod")) + { + Com_Printf("WARNING: Ignoring download %s because downloads to basemod directory are not allowed.\n", + entry->local_name); + return qfalse; + } + + if (entry_match_in_index(entry, &different_moddir_match)) + { + if (recheck_hash) + { + Com_Printf( + "WARNING: Downloaded pk3 %s has unexpected hash which already exists in index." + " Download not saved.\n", + entry->local_name); + } + return qfalse; + } + + if (!recheck_hash) + { + if (check_attempted_download(hash, qfalse)) + { + Com_Printf( + "WARNING: Ignoring download %s because a download with the same hash has already been" + " attempted in this session.\n", + entry->local_name); + return qfalse; + } + if (curl_disconnected && check_attempted_download(hash, qtrue)) + { + // Wait for the reconnect to attempt this as a UDP download + return qfalse; + } + } + + // NOTE: Consider using hash-based check instead of the old filename check? + if (fs_is_download_id_pak(entry)) + { + Com_Printf("WARNING: Ignoring download %s as possible ID pak.\n", entry->local_name); + return qfalse; + } + + if (different_moddir_match) + { + char buffer[FS_FILE_BUFFER_SIZE]; + fs_file_to_buffer((fsc_file_t *)different_moddir_match, buffer, sizeof(buffer), qfalse, qtrue, qfalse, qfalse); + Com_Printf( + "WARNING: %s %s, even though the file already appears to exist at %s." + " Set fs_redownload_across_mods to 0 to disable this behavior.\n", + recheck_hash ? "Saving" : "Downloading", entry->local_name, buffer); + } + + return qtrue; +} + +/* ******************************************************************************** */ +// Download List Creation +/* ******************************************************************************** */ + +static download_entry_t *create_download_entry(const char *name, unsigned int hash) +{ + // Returns new entry on success, null on error. + // Download entries should be freed by fs_free_download_entry. + download_entry_t *entry; + char temp_mod_dir[FSC_MAX_MODDIR]; + const char *temp_filename = 0; + char mod_dir[FSC_MAX_MODDIR]; + char filename[MAX_DOWNLOAD_NAME + 1]; + + // Generate mod_dir and filename + if (!fsc_get_leading_directory(name, temp_mod_dir, sizeof(temp_mod_dir), &temp_filename)) + return 0; + if (!temp_filename) + return 0; + fs_sanitize_mod_dir(temp_mod_dir, mod_dir); + if (!*mod_dir) + return 0; + if (!fs_generate_path(temp_filename, 0, 0, 0, 0, 0, filename, sizeof(filename))) + return 0; + + // Patch mod dir capitalization + if (!Q_stricmp(mod_dir, fs_basegame->string)) + Q_strncpyz(mod_dir, fs_basegame->string, sizeof(mod_dir)); + if (!Q_stricmp(mod_dir, FS_GetCurrentGameDir())) + Q_strncpyz(mod_dir, FS_GetCurrentGameDir(), sizeof(mod_dir)); + + // Set the download entry strings + entry = (download_entry_t *)Z_Malloc(sizeof(*entry)); + entry->local_name = + CopyString(va("%s/%s%s.pk3", mod_dir, fs_saveto_dlfolder->integer ? "downloads/" : "", filename)); + entry->remote_name = CopyString(va("%s.pk3", name)); + entry->mod_dir = CopyString(mod_dir); + entry->filename = CopyString(filename); + entry->hash = hash; + return entry; +} + +void fs_print_download_list(void) +{ + // Prints predicted needed pak list to console + download_entry_t *entry = next_download; + qboolean have_entry = qfalse; + while (entry) + { + if (!entry_match_in_index(entry, 0)) + { + if (!have_entry) + { + Com_Printf("Need paks: %s", entry->remote_name); + have_entry = qtrue; + } + else + { + Com_Printf(", %s", entry->remote_name); + } + } + entry = entry->next; + } + if (have_entry) + Com_Printf("\n"); +} + +void fs_register_download_list(const char *hash_list, const char *name_list) +{ + // This command is used to process the list of potential downloads received from the server. + int i; + int count; + int hashes[1024]; + + fs_free_download_list(); + + Cmd_TokenizeString(hash_list); + count = Cmd_Argc(); + if (count > ARRAY_LEN(hashes)) + count = ARRAY_LEN(hashes); + for (i = 0; i < count; ++i) + { + hashes[i] = atoi(Cmd_Argv(i)); + } + + Cmd_TokenizeString(name_list); + if (Cmd_Argc() < count) + count = Cmd_Argc(); + for (i = count - 1; i >= 0; --i) + { + download_entry_t *entry = create_download_entry(Cmd_Argv(i), hashes[i]); + if (!entry) + { + Com_Printf("WARNING: Ignoring download %s due to invalid name.\n", Cmd_Argv(i)); + continue; + } + fs_add_next_download(entry); + } +} + +/* ******************************************************************************** */ +// Download List Advancement +/* ******************************************************************************** */ + +void fs_advance_next_needed_download(qboolean curl_disconnected) +{ + // Advances through download queue until the current download is either null + // or valid to download (from the filesystem perspectiveat at least; CL_NextDownload + // may skip downloads for other reasons by calling fs_advance_download) + if (!current_download) + fs_advance_download(); + while (current_download) + { + if (fs_is_valid_download(current_download, 0, curl_disconnected)) + break; + fs_advance_download(); + } +} + +qboolean fs_get_current_download_info( + char **local_name_out, char **remote_name_out, qboolean *curl_already_attempted_out) +{ + // Returns qtrue and writes info if current_download is available, qfalse if it's null + if (!current_download) + return qfalse; + *local_name_out = current_download->local_name; + *remote_name_out = current_download->remote_name; + *curl_already_attempted_out = check_attempted_download(current_download->hash, qtrue); + return qtrue; +} + +int fs_estimate_remaining_downloads(void) +{ + // Tremulous - quick and dirty estimate of remaining download count for UI menu purposes + int count = 0; + download_entry_t *entry = current_download ? current_download : next_download; + + while (entry) + { + // Count downloads pending in either HTTP or UDP mode + if (fs_is_valid_download(entry, 0, qfalse) || fs_is_valid_download(entry, 0, qtrue)) + { + ++count; + } + entry = entry->next; + } + + return count; +} + +/* ******************************************************************************** */ +// Download Completion +/* ******************************************************************************** */ + +static void get_temp_file_hash_callback(void *context, char *data, int size) +{ + *(unsigned int *)context = fsc_block_checksum(data, size); +} + +static unsigned int get_temp_file_hash(const char *tempfile_path) +{ + void *os_path = fsc_string_to_os_path(tempfile_path); + unsigned int result = 0; + fsc_load_pk3(os_path, &fs, 0, 0, get_temp_file_hash_callback, &result); + fsc_free(os_path); + return result; +} + +void fs_finalize_download(void) +{ + // Does some final verification and moves the download, which hopefully has been written to + // the temporary file, to its final location. + char tempfile_path[FS_MAX_PATH]; + char target_path[FS_MAX_PATH]; + unsigned int actual_hash; + + if (!current_download) + { + // Shouldn't happen + Com_Printf("^3WARNING: fs_finalize_download called with no current download\n"); + return; + } + + if (!fs_generate_path_writedir("download.temp", 0, 0, 0, tempfile_path, sizeof(tempfile_path))) + { + Com_Printf("ERROR: Failed to get tempfile path for download\n"); + return; + } + if (!fs_generate_path_writedir(current_download->local_name, 0, + FS_ALLOW_PK3 | FS_ALLOW_DIRECTORIES | FS_CREATE_DIRECTORIES_FOR_FILE, 0, target_path, sizeof(target_path))) + { + Com_Printf("ERROR: Failed to get target path for download\n"); + return; + } + + actual_hash = get_temp_file_hash(tempfile_path); + if (!actual_hash) + { + Com_Printf("WARNING: Downloaded pk3 %s appears to be missing or corrupt. Download not saved.\n", + current_download->local_name); + return; + } + + if (actual_hash != current_download->hash) + { + // Wrong hash - this could be a malicious attempt to spoof a core pak or maybe a corrupt + // download, but probably is just a server configuration issue mixing up pak versions. + // Run the file needed check with the new hash to see if it still passes. + if (!fs_is_valid_download(current_download, actual_hash, qfalse)) + { + // Error should already be printed + return; + } + else + { + Com_Printf("WARNING: Downloaded pk3 %s has unexpected hash.\n", current_download->local_name); + } + } + + if (FS_FileInPathExists(target_path)) + { + const char *new_name = va("%s/%s%s.%08x.pk3", current_download->mod_dir, + fs_saveto_dlfolder->integer ? "downloads/" : "", current_download->filename, actual_hash); + Com_Printf("WARNING: Downloaded pk3 %s conflicts with existing file. Using name %s instead.\n", + current_download->local_name, new_name); + if (!fs_generate_path_writedir( + new_name, 0, FS_ALLOW_DIRECTORIES | FS_ALLOW_PK3, 0, target_path, sizeof(target_path))) + { + Com_Printf("ERROR: Failed to get nonconflicted target path for download\n"); + return; + } + + fs_delete_file(target_path); + } + + fs_rename_file(tempfile_path, target_path); + if (FS_FileInPathExists(tempfile_path)) + { + Com_Printf( + "ERROR: There was a problem moving downloaded pk3 %s from temporary file to target" + " location. Download may not be saved.\n", + current_download->local_name); + } + else + { + // Download appears successful; refresh filesystem to make sure it is properly registered + fs_refresh(qtrue); + } +} + +#endif // NEW_FILESYSTEM diff --git a/src/filesystem/fs_fileio.cpp b/src/filesystem/fs_fileio.cpp new file mode 100644 index 000000000..feee9dfe7 --- /dev/null +++ b/src/filesystem/fs_fileio.cpp @@ -0,0 +1,2197 @@ +/* +=========================================================================== +Copyright (C) 1999-2005 Id Software, Inc. +Copyright (C) 2017 Noah Metzger (chomenor@gmail.com) + +This file is part of Quake III Arena source code. + +Quake III Arena source code is free software; you can redistribute it +and/or modify it under the terms of the GNU General Public License as +published by the Free Software Foundation; either version 2 of the License, +or (at your option) any later version. + +Quake III Arena source code is distributed in the hope that it will be +useful, but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with Quake III Arena source code; if not, write to the Free Software +Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA +=========================================================================== +*/ + +#ifdef NEW_FILESYSTEM +#include "fslocal.h" + +/* ******************************************************************************** */ +// Path Handling Functions +/* ******************************************************************************** */ + +static void fs_mkdir(const char *directory) +{ + void *os_path = fsc_string_to_os_path(directory); + fsc_mkdir(os_path); + fsc_free(os_path); +} + +static void fs_mkdir_in_range(char *base, char *position, qboolean for_file) +{ + // Base and position should be part of the same string, where base represents + // the beginning of the string and position is the part where actual directory + // creation starts. + // If for_file is set, the final part of the path will not be created as a directory. + while (1) + { + if (*position == '/') + { + *position = 0; + fs_mkdir(base); + *position = '/'; + } + if (!*position) + { + if (!for_file) + fs_mkdir(base); + return; + } + ++position; + } +} + +static const char *fs_valid_filename_char_table(void) +{ + // Returns map of characters allowed in filenames for direct disk access, with + // invalid characters converted to underscores + // This table affects write operations and certain read operations that use + // fs_generate_path, but not the main file index + static char table[256]; + static int have_table = 0; + if (!have_table) + { + int i; + char valid_chars[] = " ~!@#$%^&_-+=()[]{}';,."; + for (i = 0; i < 256; ++i) + table[i] = '_'; + for (i = 'a'; i <= 'z'; ++i) + table[i] = i; + for (i = 'A'; i <= 'Z'; ++i) + table[i] = i; + for (i = '0'; i <= '9'; ++i) + table[i] = i; + for (i = 0; i < sizeof(valid_chars) - 1; ++i) + table[((unsigned char *)valid_chars)[i]] = valid_chars[i]; + have_table = 1; + } + return table; +} + +#define MAX_SUBPATH_LENGTH \ + 128 // Max length of each subpath in fs_generate_path (does not affect FS_NO_SANITIZE subpaths) + +static qboolean fs_generate_path_element(fsc_stream_t *stream, const char *name, int flags) +{ + // Sanitize name of single file or directory and write to stream + // Returns qtrue on success, qfalse on error + char sanitized_path[MAX_SUBPATH_LENGTH]; + int path_length; + fsc_stream_t path_stream = {sanitized_path, 0, sizeof(sanitized_path), 0}; + + // Perform character filtering + fsc_stream_append_string_substituted(&path_stream, name, fs_valid_filename_char_table()); + path_length = fsc_strlen(sanitized_path); // Should equal path_stream.position, but recalculate to be safe + if (!path_length) + return qfalse; + +// Replace non-alphanumeric characters at beginning or end of string with underscores +// Exploits against OS-specific filename processing behavior often rely on symbols at beginning or end of filename +#define IS_ALPHANUMERIC(c) (((c) >= 'a' && (c) <= 'z') || ((c) >= 'A' && (c) <= 'Z') || ((c) >= '0' && (c) <= '9')) + if (!IS_ALPHANUMERIC(sanitized_path[0])) + sanitized_path[0] = '_'; + if (!IS_ALPHANUMERIC(sanitized_path[path_length - 1])) + sanitized_path[path_length - 1] = '_'; + + // Check for possible backwards path + if (strstr(sanitized_path, "..")) + return qfalse; + + // Check for disallowed extensions + if (path_length >= 4 && !Q_stricmp(sanitized_path + path_length - 4, ".qvm")) + return qfalse; + if (path_length >= 4 && !Q_stricmp(sanitized_path + path_length - 4, ".exe")) + return qfalse; + if (path_length >= 4 && !Q_stricmp(sanitized_path + path_length - 4, ".app")) + return qfalse; + if (!(flags & FS_ALLOW_PK3) && path_length >= 4 && !Q_stricmp(sanitized_path + path_length - 4, ".pk3")) + return qfalse; + if (!(flags & FS_ALLOW_DLL)) + { + if (Sys_DllExtension(sanitized_path)) + return qfalse; + // Do some extra checks to be safe + if (path_length >= 4 && !Q_stricmp(sanitized_path + path_length - 4, ".dll")) + return qfalse; + if (path_length >= 3 && !Q_stricmp(sanitized_path + path_length - 3, ".so")) + return qfalse; + if (path_length >= 6 && !Q_stricmp(sanitized_path + path_length - 6, ".dylib")) + return qfalse; + } + if (!(flags & FS_ALLOW_SPECIAL_CFG) && + (!Q_stricmp(sanitized_path, Q3CONFIG_CFG) || !Q_stricmp(sanitized_path, "autoexec.cfg"))) + return qfalse; + + // Write out the string + fsc_stream_append_string(stream, sanitized_path); + return qtrue; +} + +static qboolean fs_generate_subpath(fsc_stream_t *stream, const char *path, int flags) +{ + // Writes path to stream with sanitization and other operations based on flags + // Returns qtrue on success, qfalse on error + int old_position = stream->position; + + if (flags & FS_NO_SANITIZE) + { + // If sanitize disabled, just write out the string + fsc_stream_append_string(stream, path); + } + + else if (flags & FS_ALLOW_DIRECTORIES) + { + // Write each section of the path separated by slashes + char name[MAX_SUBPATH_LENGTH]; + const char *path_ptr = path; + qboolean first_element = qtrue; + if (fsc_strlen(path) >= MAX_SUBPATH_LENGTH) + return qfalse; + while (path_ptr) + { + if (!fsc_get_leading_directory(path_ptr, name, sizeof(name), &path_ptr)) + { + // Ignore empty sections caused by excess slashes + continue; + } + if (!first_element) + fsc_stream_append_string(stream, "/"); + if (!fs_generate_path_element(stream, name, flags)) + { + // Abort on sanitize error + return qfalse; + } + first_element = qfalse; + } + } + + else + { + // Write single path element + if (!fs_generate_path_element(stream, path, flags)) + return qfalse; + } + + // Create directories for path + if (flags & FS_CREATE_DIRECTORIES_FOR_FILE) + { + fs_mkdir_in_range(stream->data, stream->data + old_position, qtrue); + } + else if (flags & FS_CREATE_DIRECTORIES) + { + fs_mkdir_in_range(stream->data, stream->data + old_position, qfalse); + } + + return qtrue; +} + +unsigned int fs_generate_path(const char *path1, const char *path2, const char *path3, int path1_flags, int path2_flags, + int path3_flags, char *target, unsigned int target_size) +{ + // Concatenates paths, adding '/' character as seperator, with sanitization + // and directory creation based on flags + // Returns output length on success, 0 on error (overflow or sanitize error) + fsc_stream_t stream = {target, 0, target_size, 0}; + FSC_ASSERT(target); + + if (path1) + { + if (!fs_generate_subpath(&stream, path1, path1_flags)) + goto error; + } + + if (path2) + { + if (path1) + fsc_stream_append_string(&stream, "/"); + if (!fs_generate_subpath(&stream, path2, path2_flags)) + goto error; + } + + if (path3) + { + if (path1 || path2) + fsc_stream_append_string(&stream, "/"); + if (!fs_generate_subpath(&stream, path3, path3_flags)) + goto error; + } + + if (!stream.position || stream.overflowed) + goto error; + return stream.position; + +error: + *target = 0; + return 0; +} + +unsigned int fs_generate_path_sourcedir(int source_dir_id, const char *path1, const char *path2, int path1_flags, + int path2_flags, char *target, unsigned int target_size) +{ + // Generates path prefixed by source directory + FSC_ASSERT(target); + if (!fs_sourcedirs[source_dir_id].active) + { + *target = 0; + return 0; + } + return fs_generate_path( + fs_sourcedirs[source_dir_id].path, path1, path2, FS_NO_SANITIZE, path1_flags, path2_flags, target, target_size); +} + +unsigned int fs_generate_path_writedir( + const char *path1, const char *path2, int path1_flags, int path2_flags, char *target, unsigned int target_size) +{ + // Generates path prefixed by write directory + FSC_ASSERT(target); + if (fs_read_only) + { + *target = 0; + return 0; + } + return fs_generate_path_sourcedir(0, path1, path2, path1_flags, path2_flags, target, target_size); +} + +/* ******************************************************************************** */ +// Direct file access functions +/* ******************************************************************************** */ + +void *fs_open_file(const char *path, const char *mode) +{ + // Standard string version of fsc_open_file + FSC_ASSERT(path && mode); + { + void *os_path = fsc_string_to_os_path(path); + void *handle = fsc_open_file(os_path, mode); + fsc_free(os_path); + return handle; + } +} + +void fs_rename_file(const char *source, const char *target) +{ + // Standard string version of fsc_rename_file + FSC_ASSERT(source && target); + { + void *source_os_path = fsc_string_to_os_path(source); + void *target_os_path = fsc_string_to_os_path(target); + fsc_rename_file(source_os_path, target_os_path); + fsc_free(source_os_path); + fsc_free(target_os_path); + } +} + +void fs_delete_file(const char *path) +{ + // Standard string version of fsc_delete_file + FSC_ASSERT(path); + { + void *os_path = fsc_string_to_os_path(path); + fsc_delete_file(os_path); + fsc_free(os_path); + } +} + +void FS_HomeRemove(const char *homePath) +{ + char path[FS_MAX_PATH]; + if (!fs_generate_path_writedir(FS_GetCurrentGameDir(), homePath, 0, FS_ALLOW_DIRECTORIES, path, sizeof(path))) + { + Com_Printf("WARNING: FS_HomeRemove on %s failed due to invalid path\n", homePath); + return; + } + + fs_delete_file(path); +} + +qboolean FS_FileInPathExists(const char *testpath) +{ + void *handle = fs_open_file(testpath, "rb"); + if (handle) + { + fsc_fclose(handle); + return qtrue; + } + return qfalse; +} + +qboolean FS_FileExists(const char *file) +{ + char path[FS_MAX_PATH]; + if (!fs_generate_path_sourcedir(0, FS_GetCurrentGameDir(), file, 0, FS_ALLOW_DIRECTORIES, path, sizeof(path))) + return qfalse; + return FS_FileInPathExists(path); +} + +static const char *fs_tremulous_write_mod_location(fs_handle_owner_t owner); +qboolean FS_FileExistsInBaseGame(const char *file) +{ + char path[FS_MAX_PATH]; + if (!fs_generate_path_sourcedir(0, fs_tremulous_write_mod_location(FS_HANDLEOWNER_SYSTEM), file, 0, + FS_ALLOW_DIRECTORIES, path, sizeof(path))) + return qfalse; + return FS_FileInPathExists(path); +} + +/* ******************************************************************************** */ +// File read cache +/* ******************************************************************************** */ + +typedef struct cache_entry_s { + unsigned int size; + int lock_count; + int stage; + + const fsc_file_t *file; + unsigned int file_size; + unsigned int file_timestamp; + + struct cache_entry_s *next_position; + unsigned int lookup_hash; + struct cache_entry_s *next_lookup; + struct cache_entry_s *prev_lookup; +} cache_entry_t; + +// ***** Cache lookup table ***** + +#define CACHE_LOOKUP_TABLE_SIZE 4096 +static cache_entry_t *cache_lookup_table[CACHE_LOOKUP_TABLE_SIZE]; + +static unsigned int cache_hash(const fsc_file_t *file) +{ + if (!file) + return 0; + return fsc_string_hash((const char *)STACKPTR(file->qp_name_ptr), (const char *)STACKPTRN(file->qp_dir_ptr)); +} + +static void cache_lookup_table_register(cache_entry_t *entry) +{ + int position = entry->lookup_hash % CACHE_LOOKUP_TABLE_SIZE; + entry->next_lookup = cache_lookup_table[position]; + entry->prev_lookup = 0; + if (cache_lookup_table[position]) + cache_lookup_table[position]->prev_lookup = entry; + cache_lookup_table[position] = entry; +} + +static void cache_lookup_table_deregister(cache_entry_t *entry) +{ + int position = entry->lookup_hash % CACHE_LOOKUP_TABLE_SIZE; + if (entry->next_lookup) + entry->next_lookup->prev_lookup = entry->prev_lookup; + if (entry->prev_lookup) + entry->prev_lookup->next_lookup = entry->next_lookup; + else + cache_lookup_table[position] = entry->next_lookup; +} + +static void cache_lookup_table_deregister_range(cache_entry_t *start, cache_entry_t *end) +{ + while (start && start != end) + { + cache_lookup_table_deregister(start); + start = start->next_position; + } +} + +static qboolean cache_entry_matches_file(const fsc_file_t *file, const cache_entry_t *cache_entry) +{ + // We need a little more extensive check than just comparing the file pointer, + // because fsc_load_file can reuse an existing file object for a modified file in specific cases + if (file != cache_entry->file) + return qfalse; + if (cache_entry->file_size != cache_entry->file->filesize) + return qfalse; + if (file->sourcetype == FSC_SOURCETYPE_DIRECT && + cache_entry->file_timestamp != ((fsc_file_direct_t *)file)->os_timestamp) + return qfalse; + return qtrue; +} + +static cache_entry_t *cache_lookup_search(const fsc_file_t *file) +{ + cache_entry_t *entry = cache_lookup_table[cache_hash(file) % CACHE_LOOKUP_TABLE_SIZE]; + cache_entry_t *best_entry = 0; + + while (entry) + { + if ((!best_entry || entry->stage > best_entry->stage) && cache_entry_matches_file(file, entry)) + { + best_entry = entry; + } + entry = entry->next_lookup; + } + + return best_entry; +} + +// ***** Cache data store ***** + +#define CACHE_ENTRY_DATA(cache_entry) ((char *)(cache_entry) + sizeof(cache_entry_t)) +#define CACHE_ALIGN(ptr) (((uintptr_t)(ptr) + 15) & ~15) + +int cache_stage = 0; +int cache_size = 0; +cache_entry_t *base_entry; +cache_entry_t *head_entry; // Last entry created. Null if just initialized. + +static cache_entry_t *cache_allocate(const fsc_file_t *file, unsigned int size) +{ + unsigned int required_space = size + sizeof(cache_entry_t); + int wrapped_around = 0; + cache_entry_t *lead_entry = head_entry; // Entry preceding new entry (can be null) + cache_entry_t *limit_entry = lead_entry ? lead_entry->next_position : 0; // Entry following new entry (can be null) + char *start_point, *end_point; // Range of memory available for new entry: [start, end) + cache_entry_t *new_entry; + + while (1) + { + // Check if we have enough space yet + start_point = (char *)CACHE_ALIGN( + lead_entry ? (char *)lead_entry + lead_entry->size + sizeof(cache_entry_t) : (char *)base_entry); + end_point = limit_entry ? (char *)limit_entry : (char *)base_entry + cache_size; + if (end_point < start_point) + { + Com_Error(ERR_FATAL, "fscache buffer position fault"); + } + if (end_point - start_point >= required_space) + break; + + // Wraparound check + if (!limit_entry) + { + if (!head_entry || wrapped_around++) + return 0; + lead_entry = 0; + limit_entry = base_entry; + continue; + } + + // Don't advance limit over a locked entry + while (limit_entry && limit_entry->lock_count) + { + lead_entry = limit_entry; + limit_entry = lead_entry->next_position; + } + + // Advance limit + if (limit_entry) + limit_entry = limit_entry->next_position; + } + + // We have space for a new entry + new_entry = (cache_entry_t *)start_point; + + // Deregister entries we are overwriting + if (lead_entry) + { + cache_lookup_table_deregister_range(lead_entry->next_position, limit_entry); + lead_entry->next_position = new_entry; + } + else if (head_entry) + { + cache_lookup_table_deregister_range(base_entry, limit_entry); + } + + new_entry->next_position = limit_entry; + head_entry = new_entry; + + new_entry->size = size; + new_entry->lock_count = 0; + new_entry->stage = cache_stage; + new_entry->file = file; + new_entry->file_size = file ? file->filesize : 0; + new_entry->file_timestamp = + file && file->sourcetype == FSC_SOURCETYPE_DIRECT ? ((fsc_file_direct_t *)file)->os_timestamp : 0; + new_entry->lookup_hash = cache_hash(file); + + cache_lookup_table_register(new_entry); + + return new_entry; +} + +void fs_cache_initialize(void) +{ + // This gets called directly from Com_Init, after the config files have been read, + // to allow setting fs_read_cache_megs in the normal config file instead of the command line. +#ifdef DEDICATED + cvar_t *cache_megs_cvar = Cvar_Get("fs_read_cache_megs", "4", CVAR_LATCH | CVAR_ARCHIVE); +#else + cvar_t *cache_megs_cvar = Cvar_Get("fs_read_cache_megs", "64", CVAR_LATCH | CVAR_ARCHIVE); +#endif + int cache_megs = cache_megs_cvar->integer; + if (cache_megs < 0) + cache_megs = 0; + if (cache_megs > 1024) + cache_megs = 1024; + + cache_size = cache_megs << 20; + base_entry = (cache_entry_t *)fsc_malloc(cache_size); + head_entry = 0; +} + +void fs_advance_cache_stage(void) +{ + // Causes existing files in cache to be recopied to the front of the cache on reference + // This may be called between level loads to help with performance + // It should not have any functional impact; it is only for optimization purposes + ++cache_stage; +} + +// ***** Cache debugging ***** + +static int fs_cache_entrycount_direct(void) +{ + cache_entry_t *entry = base_entry; + int count = 0; + + if (!head_entry) + return 0; + do + { + ++count; + } while ((entry = entry->next_position)); + + return count; +} + +static int fs_cache_entrycount_lookuptable(void) +{ + int i; + cache_entry_t *entry; + int count = 0; + + for (i = 0; i < CACHE_LOOKUP_TABLE_SIZE; ++i) + { + entry = cache_lookup_table[i]; + while (entry) + { + ++count; + entry = entry->next_lookup; + } + } + + return count; +} + +void fs_readcache_debug(void) +{ + // Debug prints the contents of the cache + cache_entry_t *entry = base_entry; + char data[1000]; + fsc_stream_t stream = {data, 0, sizeof(data), 0}; + int index_counter = 0; + + if (!head_entry) + return; + +#define ADD_STRING(string) fsc_stream_append_string(&stream, string) + + do + { + stream.position = 0; + if (!entry->file) + { + ADD_STRING(va("Null File Index(%i) Position(%i) Size(%i) Stage(%i) Lockcount(%i)", index_counter, + (int)((char *)entry - (char *)base_entry), entry->size, entry->stage, entry->lock_count)); + } + else + { + ADD_STRING("File("); + fs_file_to_stream(entry->file, &stream, qtrue, qtrue, qtrue, qfalse); + ADD_STRING(va(") Index(%i) Position(%i) Size(%i) Stage(%i) Lockcount(%i)", index_counter, + (int)((char *)entry - (char *)base_entry), entry->size, entry->stage, entry->lock_count)); + } + if (entry == head_entry) + ADD_STRING(" "); + ADD_STRING("\n\n"); + Com_Printf("%s", stream.data); + ++index_counter; + } while ((entry = entry->next_position)); + + Com_Printf("entry count from direct iteration: %i\n", fs_cache_entrycount_direct()); + Com_Printf("entry count from lookup table: %i\n", fs_cache_entrycount_lookuptable()); +} + +/* ******************************************************************************** */ +// Data reading +/* ******************************************************************************** */ + +static cache_entry_t *cache_search_current_stage(const fsc_file_t *file) +{ + // Performs cache search, and if entry doesn't match current stage, try to create current stage duplicate. + cache_entry_t *entry = cache_lookup_search(file); + if (!entry) + return 0; + + if (entry->stage != cache_stage) + { + cache_entry_t *new_entry; + ++entry->lock_count; + new_entry = cache_allocate(file, entry->size); + --entry->lock_count; + if (new_entry) + { + fsc_memcpy(CACHE_ENTRY_DATA(new_entry), CACHE_ENTRY_DATA(entry), entry->size); + return new_entry; + } + } + + return entry; +} + +char *fs_read_data(const fsc_file_t *file, const char *path, unsigned int *size_out, const char *calling_function) +{ + // Input can be either file or path, not both. + // Returns null on error, otherwise result needs to be freed by fs_free_data. + // Currently file-type read always reads file->filesize, otherwise it is an error and null is returned. + cache_entry_t *cache_entry = 0; + char *data = 0; + void *os_path = 0; + void *fsc_file_handle = 0; + unsigned int size; + + // Ensure we have file or path set but not both + if ((file && path) || (!file && !path)) + Com_Error(ERR_DROP, "Invalid parameters to fs_read_data."); + + // Mark the file in reference tracking + if (file) + fs_register_reference(file); + + // Print leading debug info + if (fs_debug_fileio->integer) + { + FS_DPrintf("********** load file data **********\n"); + FS_DPrintf(" origin: %s\n", calling_function); + if (file) + { + char buffer[FS_FILE_BUFFER_SIZE]; + fs_file_to_buffer(file, buffer, sizeof(buffer), qtrue, qtrue, qtrue, qfalse); + FS_DPrintf(" type: indexed file\n"); + FS_DPrintf(" file: %s\n", buffer); + } + else + { + FS_DPrintf(" type: direct path\n"); + FS_DPrintf(" path: %s\n", path); + } + } + + // Check if file is already available from cache + if (file) + { + cache_entry = cache_search_current_stage(file); + if (cache_entry) + { + ++cache_entry->lock_count; + if (size_out) + *size_out = cache_entry->size - 1; + if (fs_debug_fileio->integer) + FS_DPrintf(" result: loaded %u bytes from cache\n", cache_entry->size - 1); + return CACHE_ENTRY_DATA(cache_entry); + } + } + + // Derive os_path in case of path parameter or direct sourcetype file + if (path) + { + os_path = fsc_string_to_os_path(path); + if (!os_path) + goto error; + } + else if (file && file->sourcetype == FSC_SOURCETYPE_DIRECT) + os_path = STACKPTR(((fsc_file_direct_t *)file)->os_path_ptr); + + // Obtain handle (if applicable) and size + if (os_path) + { + fsc_file_handle = fsc_open_file(os_path, "rb"); + if (path) + fsc_free(os_path); + if (!fsc_file_handle) + goto error; + + fsc_fseek(fsc_file_handle, 0, FSC_SEEK_END); + size = fsc_ftell(fsc_file_handle); + + fsc_fseek(fsc_file_handle, 0, FSC_SEEK_SET); + } + else + size = file->filesize; + + // Set a file size limit of about 2GB as a catch-all to avoid overflow conditions + // The game shouldn't normally need to read such big files using this function + if (size > 2000000000) + { + Com_Printf("WARNING: Excessive file size in fs_read_data\n"); + goto error; + } + + // Obtain buffer from cache or malloc + if (size < cache_size / 3) + { + // Don't use more than 1/3 of the cache for a single file to avoid flushing smaller files + cache_entry = cache_allocate(file, size + 1); + } + if (cache_entry) + { + ++cache_entry->lock_count; + data = CACHE_ENTRY_DATA(cache_entry); + } + else + data = (char *)fsc_malloc(size + 1); + + // Extract data into buffer + if (fsc_file_handle) + { + int read_size = fsc_fread(data, size + 1, fsc_file_handle); + fsc_fclose(fsc_file_handle); + if (read_size != size) + goto error; + } + else + { + if (fsc_extract_file(file, data, &fs, 0)) + goto error; + } + data[size] = 0; + + if (size_out) + *size_out = size; + if (fs_debug_fileio->integer) + FS_DPrintf(" result: loaded %u bytes from file\n", size); + return data; + +// Free buffer if there was an error extracting data +error: + if (fs_debug_fileio->integer) + FS_DPrintf(" result: failed to load file\n"); + if (cache_entry) + { + cache_entry->file = 0; + cache_entry->lock_count = 0; + } + else if (data) + fsc_free(data); + if (size_out) + *size_out = 0; + return 0; +} + +void fs_free_data(char *data) +{ + FSC_ASSERT(data); + if (data >= (char *)base_entry && data < (char *)base_entry + cache_size) + { + cache_entry_t *cache_entry = (cache_entry_t *)(data - sizeof(cache_entry_t)); + if (cache_entry->lock_count <= 0) + Com_Error(ERR_DROP, "fs_free_data on invalid or already freed entry."); + --cache_entry->lock_count; + } + else + { + fsc_free(data); + } +} + +char *fs_read_shader(const fsc_shader_t *shader) +{ + // Returns shader text allocated in Z_Malloc, or null if there was an error + unsigned int size; + char *source_data; + char *shader_data; + FSC_ASSERT(shader); + + if (fs_debug_fileio->integer) + { + FS_DPrintf("********** read shader **********\n"); + fs_debug_indent_start(); + FS_DPrintf("name: %s\n", (char *)STACKPTR(shader->shader_name_ptr)); + } + + size = shader->end_position - shader->start_position; + if (size > 10000) + { + if (fs_debug_fileio->integer) + { + FS_DPrintf("result: failed due to invalid size\n"); + fs_debug_indent_stop(); + } + return 0; + } + + source_data = fs_read_data((const fsc_file_t *)STACKPTR(shader->source_file_ptr), 0, 0, "fs_read_shader"); + if (!source_data) + { + if (fs_debug_fileio->integer) + { + FS_DPrintf("result: failed to read source file\n"); + fs_debug_indent_stop(); + } + return 0; + } + + shader_data = (char *)Z_Malloc(size + 1); + fsc_memcpy(shader_data, source_data + shader->start_position, size); + shader_data[size] = 0; + + fs_free_data(source_data); + + if (fs_debug_fileio->integer) + { + FS_DPrintf("result: loaded %i shader bytes\n", size); + fs_debug_indent_stop(); + } + + return shader_data; +} + +/* ******************************************************************************** */ +// File handle management +/* ******************************************************************************** */ + +typedef enum { + FS_HANDLE_INVALID, + FS_HANDLE_CACHE_READ, + FS_HANDLE_DIRECT_READ, + FS_HANDLE_PK3_READ, + FS_HANDLE_WRITE, + FS_HANDLE_PIPE +} fs_handle_type_t; + +typedef struct { + fs_handle_type_t type; + fs_handle_owner_t owner; + char *debug_path; +} fs_handle_t; + +typedef struct { + fs_handle_t h; + char *data; + unsigned int position; + unsigned int size; +} fs_cache_read_handle_t; + +typedef struct { + fs_handle_t h; + void *fsc_handle; +} fs_direct_read_handle_t; + +typedef struct { + fs_handle_t h; + const fsc_file_frompk3_t *file; + void *fsc_handle; + unsigned int position; +} fs_pk3_read_handle_t; + +typedef struct { + fs_handle_t h; + void *fsc_handle; + qboolean sync; +} fs_write_handle_t; + +typedef struct { + fs_handle_t h; + FILE *handle; +} fs_pipe_handle_t; + +// Note fileHandle_t values are incremented by 1 compared to the indices in this array. +#define MAX_HANDLES 64 +fs_handle_t *handles[MAX_HANDLES]; + +static fileHandle_t fs_allocate_handle(fs_handle_type_t type) +{ + int index; + int size; + + // Locate free handle + for (index = 0; index < MAX_HANDLES; ++index) + { + if (!handles[index]) + break; + } + if (index >= MAX_HANDLES) + Com_Error(ERR_DROP, "fs_allocate_handle failed to find free handle"); + + // Get size + switch (type) + { + case FS_HANDLE_CACHE_READ: + size = sizeof(fs_cache_read_handle_t); + break; + case FS_HANDLE_DIRECT_READ: + size = sizeof(fs_direct_read_handle_t); + break; + case FS_HANDLE_PK3_READ: + size = sizeof(fs_pk3_read_handle_t); + break; + case FS_HANDLE_WRITE: + size = sizeof(fs_write_handle_t); + break; + case FS_HANDLE_PIPE: + size = sizeof(fs_pipe_handle_t); + break; + default: + Com_Error(ERR_DROP, "fs_allocate_handle with invalid type"); + } + + // Allocate + handles[index] = (fs_handle_t *)Z_Malloc(size); + handles[index]->type = type; + handles[index]->owner = FS_HANDLEOWNER_SYSTEM; + return index + 1; +} + +static void fs_free_handle(fileHandle_t handle) +{ + int index = handle - 1; + if (index < 0 || index > MAX_HANDLES || !handles[index]) + Com_Error(ERR_DROP, "fs_free_handle on invalid handle"); + Z_Free(handles[index]); + handles[index] = 0; +} + +static fs_handle_t *fs_get_handle_entry(fileHandle_t handle) +{ + // Returns null if handle is invalid + int index = handle - 1; + if (index < 0 || index > MAX_HANDLES || !handles[index]) + return 0; + return handles[index]; +} + +/* ******************************************************************************** */ +// Cache read handle operations +/* ******************************************************************************** */ + +static fileHandle_t fs_cache_read_handle_open(const fsc_file_t *file, const char *path, unsigned int *size_out) +{ + // Only file or path should be set, not both + // Does not include sanity check on path + // Returns handle on success, null on error + + fileHandle_t handle = fs_allocate_handle(FS_HANDLE_CACHE_READ); + fs_cache_read_handle_t *handle_entry = (fs_cache_read_handle_t *)fs_get_handle_entry(handle); + + // Set up handle entry + handle_entry->data = fs_read_data(file, path, &handle_entry->size, "fs_cache_read_handle_open"); + if (!handle_entry->data) + { + fs_free_handle(handle); + return 0; + } + handle_entry->position = 0; + + // Set debug path + if (file) + { + char buffer[FS_FILE_BUFFER_SIZE]; + fs_file_to_buffer(file, buffer, sizeof(buffer), qtrue, qtrue, qtrue, qfalse); + handle_entry->h.debug_path = CopyString(buffer); + } + else + { + handle_entry->h.debug_path = CopyString(path); + } + + if (size_out) + *size_out = handle_entry->size; + return handle; +} + +static unsigned int fs_cache_read_handle_read(char *buffer, unsigned int length, fs_cache_read_handle_t *handle_entry) +{ + // Don't read past end of file... + if (length > handle_entry->size - handle_entry->position) + length = handle_entry->size - handle_entry->position; + + // Read data to buffer and advance position + fsc_memcpy(buffer, handle_entry->data + handle_entry->position, length); + handle_entry->position += length; + return length; +} + +static int fs_cache_read_handle_seek(fs_cache_read_handle_t *handle_entry, int offset, fsOrigin_t origin_mode) +{ + unsigned int origin; + unsigned int offset_origin; + + // Get origin + switch (origin_mode) + { + case FS_SEEK_CUR: + origin = handle_entry->position; + break; + case FS_SEEK_END: + origin = handle_entry->size; + break; + case FS_SEEK_SET: + origin = 0; + break; + default: + Com_Error(ERR_DROP, "fs_cache_read_handle_seek with invalid origin mode"); + } + + // Get offset_origin and correct overflow conditions + offset_origin = origin + offset; + if (offset < 0 && offset_origin > origin) + offset_origin = 0; + if ((offset > 0 && offset_origin < origin) || offset_origin > handle_entry->size) + offset_origin = handle_entry->size; + + // Write the new position + handle_entry->position = offset_origin; + + if (offset_origin == origin + offset) + return 0; + return -1; +} + +static void fs_cache_read_handle_close(fs_cache_read_handle_t *handle_entry) { fs_free_data(handle_entry->data); } + +/* ******************************************************************************** */ +// Direct read handle operations +/* ******************************************************************************** */ + +fileHandle_t fs_direct_read_handle_open(const fsc_file_t *file, const char *path, unsigned int *size_out) +{ + // Only file or path should be set, not both + // Does not include sanity check on path + // Returns handle on success, null on error + char debug_path[FS_MAX_PATH]; + void *os_path; + void *fsc_handle; + fileHandle_t handle; + fs_direct_read_handle_t *handle_entry; + + if (file) + { + if (file->sourcetype != FSC_SOURCETYPE_DIRECT) + Com_Error(ERR_FATAL, "fs_direct_read_handle_open on non direct file"); + os_path = STACKPTR(((fsc_file_direct_t *)file)->os_path_ptr); + fs_file_to_buffer((fsc_file_t *)file, debug_path, sizeof(debug_path), qtrue, qtrue, qtrue, qfalse); + } + else if (path) + { + os_path = fsc_string_to_os_path(path); + Q_strncpyz(debug_path, path, sizeof(debug_path)); + } + else + Com_Error(ERR_FATAL, "Invalid parameters to fs_direct_read_handle_open."); + + if (fs_debug_fileio->integer) + { + FS_DPrintf("********** opening direct read handle **********\n"); + FS_DPrintf(" path: %s\n", debug_path); + } + + fsc_handle = fsc_open_file(os_path, "rb"); + if (!file) + fsc_free(os_path); + if (!fsc_handle) + { + if (fs_debug_fileio->integer) + FS_DPrintf(" result: failed to open file\n"); + return 0; + } + + // Set up handle entry + handle = fs_allocate_handle(FS_HANDLE_DIRECT_READ); + handle_entry = (fs_direct_read_handle_t *)fs_get_handle_entry(handle); + handle_entry->fsc_handle = fsc_handle; + handle_entry->h.debug_path = CopyString(debug_path); + + // Get size + if (size_out) + { + fsc_fseek(fsc_handle, 0, FSC_SEEK_END); + *size_out = fsc_ftell(fsc_handle); + fsc_fseek(fsc_handle, 0, FSC_SEEK_SET); + } + + if (fs_debug_fileio->integer) + FS_DPrintf(" result: success\n"); + return handle; +} + +static unsigned int fs_direct_read_handle_read(char *buffer, unsigned int length, fs_direct_read_handle_t *handle_entry) +{ + return fsc_fread(buffer, length, handle_entry->fsc_handle); +} + +static int fs_direct_read_handle_seek(fs_direct_read_handle_t *handle_entry, int offset, fsOrigin_t origin_mode) +{ + // Get type + fsc_seek_type_t type; + switch (origin_mode) + { + case FS_SEEK_CUR: + type = FSC_SEEK_CUR; + break; + case FS_SEEK_END: + type = FSC_SEEK_END; + break; + case FS_SEEK_SET: + type = FSC_SEEK_SET; + break; + default: + Com_Error(ERR_DROP, "fs_direct_read_handle_seek with invalid origin mode"); + } + return fsc_fseek(handle_entry->fsc_handle, offset, type); +} + +static void fs_direct_read_handle_close(fs_direct_read_handle_t *handle_entry) { fsc_fclose(handle_entry->fsc_handle); } + +/* ******************************************************************************** */ +// Pk3 read handle operations +/* ******************************************************************************** */ + +static fileHandle_t fs_pk3_read_handle_open(const fsc_file_t *file) +{ + // Returns handle on success, null on error + char debug_path[FS_MAX_PATH]; + void *fsc_handle; + fileHandle_t handle; + fs_pk3_read_handle_t *handle_entry; + + if (file->sourcetype != FSC_SOURCETYPE_PK3) + Com_Error(ERR_FATAL, "fs_pk3_read_handle_open on non pk3 file"); + fs_file_to_buffer((fsc_file_t *)file, debug_path, sizeof(debug_path), qtrue, qtrue, qtrue, qfalse); + + if (fs_debug_fileio->integer) + { + FS_DPrintf("********** opening pk3 read handle **********\n"); + FS_DPrintf(" file: %s\n", debug_path); + } + + fsc_handle = fsc_pk3_handle_open((fsc_file_frompk3_t *)file, 16384, &fs, 0); + if (!fsc_handle) + { + if (fs_debug_fileio->integer) + FS_DPrintf(" result: failed to open file\n"); + return 0; + } + + // Set up handle entry + handle = fs_allocate_handle(FS_HANDLE_PK3_READ); + handle_entry = (fs_pk3_read_handle_t *)fs_get_handle_entry(handle); + handle_entry->file = (fsc_file_frompk3_t *)file; + handle_entry->fsc_handle = fsc_handle; + handle_entry->position = 0; + handle_entry->h.debug_path = CopyString(debug_path); + + if (fs_debug_fileio->integer) + FS_DPrintf(" result: success\n"); + return handle; +} + +static unsigned int fs_pk3_read_handle_read(char *buffer, unsigned int length, fs_pk3_read_handle_t *handle_entry) +{ + unsigned int max_length = handle_entry->file->f.filesize - handle_entry->position; + if (length > max_length) + length = max_length; + length = fsc_pk3_handle_read(handle_entry->fsc_handle, buffer, length); + handle_entry->position += length; + return length; +} + +static int fs_pk3_read_handle_seek(fs_pk3_read_handle_t *handle_entry, int offset, fsOrigin_t origin_mode) +{ + // Uses very similar, not very efficient, method as the original filesystem + // This function is very rarely used but needs to be supported for mod compatibility + unsigned int origin; + unsigned int offset_origin; + + // Get origin + switch (origin_mode) + { + case FS_SEEK_CUR: + origin = handle_entry->position; + break; + case FS_SEEK_END: + origin = handle_entry->file->f.filesize; + break; + case FS_SEEK_SET: + origin = 0; + break; + default: + Com_Error(ERR_DROP, "fs_pk3_read_handle_seek with invalid origin mode"); + } + + // Get offset_origin and correct overflow conditions + offset_origin = origin + offset; + if (offset < 0 && offset_origin > origin) + offset_origin = 0; + if ((offset > 0 && offset_origin < origin) || offset_origin > handle_entry->file->f.filesize) + offset_origin = handle_entry->file->f.filesize; + + // If seeking to end, just set the position + if (offset_origin >= handle_entry->file->f.filesize) + { + handle_entry->position = handle_entry->file->f.filesize; + return 0; + } + + // If seeking backwards, reset the handle + if (offset_origin < handle_entry->position) + { + fsc_pk3_handle_close(handle_entry->fsc_handle); + handle_entry->fsc_handle = fsc_pk3_handle_open(handle_entry->file, 16384, &fs, 0); + if (!handle_entry->fsc_handle) + Com_Error(ERR_FATAL, "fs_pk3_read_handle_seek failed to reopen handle"); + handle_entry->position = 0; + } + + // Seek forwards by reading data to a temp buffer + while (handle_entry->position < offset_origin) + { + char buffer[65536]; + unsigned int read_amount; + unsigned int read_target = offset_origin - handle_entry->position; + if (read_target > sizeof(buffer)) + read_target = sizeof(buffer); + read_amount = fsc_pk3_handle_read(handle_entry->fsc_handle, buffer, read_target); + handle_entry->position += read_amount; + if (read_amount != read_target) + return -1; + } + + if (offset_origin == origin + offset) + return 0; + return -1; +} + +static void fs_pk3_read_handle_close(fs_pk3_read_handle_t *handle_entry) +{ + fsc_pk3_handle_close(handle_entry->fsc_handle); +} + +/* ******************************************************************************** */ +// Write handle operations +/* ******************************************************************************** */ + +static fileHandle_t fs_write_handle_open(const char *path, qboolean append, qboolean sync) +{ + // Does not include directory creation or sanity checks + // Returns handle on success, null on error + fileHandle_t handle; + fs_write_handle_t *handle_entry; + void *os_path = fsc_string_to_os_path(path); + void *fsc_handle; + + if (fs_debug_fileio->integer) + { + FS_DPrintf("********** opening write handle **********\n"); + FS_DPrintf(" path: %s\n", path); + } + + if (!os_path) + { + if (fs_debug_fileio->integer) + FS_DPrintf(" result: failed to convert to os path\n"); + return 0; + } + + // Attempt to open the file + if (append) + fsc_handle = fsc_open_file(os_path, "ab"); + else + fsc_handle = fsc_open_file(os_path, "wb"); + fsc_free(os_path); + if (!fsc_handle) + { + if (fs_debug_fileio->integer) + FS_DPrintf(" result: failed to open file\n"); + return 0; + } + + // Set up handle entry + handle = fs_allocate_handle(FS_HANDLE_WRITE); + handle_entry = (fs_write_handle_t *)fs_get_handle_entry(handle); + handle_entry->fsc_handle = fsc_handle; + handle_entry->sync = sync; + handle_entry->h.debug_path = CopyString(path); + if (fs_debug_fileio->integer) + FS_DPrintf(" result: success\n"); + return handle; +} + +static void fs_write_handle_write(fileHandle_t handle, const char *data, unsigned int length) +{ + fs_write_handle_t *handle_entry = (fs_write_handle_t *)fs_get_handle_entry(handle); + if (!handle_entry || handle_entry->h.type != FS_HANDLE_WRITE) + Com_Error(ERR_DROP, "fs_write_handle_write on invalid handle"); + + fsc_fwrite(data, length, handle_entry->fsc_handle); + if (handle_entry->sync) + fsc_fflush(handle_entry->fsc_handle); +} + +static void fs_write_handle_flush(fileHandle_t handle, qboolean enable_sync) +{ + fs_write_handle_t *handle_entry = (fs_write_handle_t *)fs_get_handle_entry(handle); + if (!handle_entry || handle_entry->h.type != FS_HANDLE_WRITE) + Com_Error(ERR_DROP, "fs_write_handle_flush on invalid handle"); + + fsc_fflush(handle_entry->fsc_handle); + if (enable_sync) + handle_entry->sync = qtrue; +} + +static int fs_write_handle_seek(fs_write_handle_t *handle_entry, int offset, fsOrigin_t origin_mode) +{ + // Get type + fsc_seek_type_t type; + switch (origin_mode) + { + case FS_SEEK_CUR: + type = FSC_SEEK_CUR; + break; + case FS_SEEK_END: + type = FSC_SEEK_END; + break; + case FS_SEEK_SET: + type = FSC_SEEK_SET; + break; + default: + Com_Error(ERR_DROP, "fs_write_handle_seek with invalid origin mode"); + } + return fsc_fseek(handle_entry->fsc_handle, offset, type); +} + +static void fs_write_handle_close(fs_write_handle_t *handle_entry) { fsc_fclose(handle_entry->fsc_handle); } + +/* ******************************************************************************** */ +// Pipe Files +/* ******************************************************************************** */ + +fileHandle_t FS_FCreateOpenPipeFile(const char *filename) +{ + char path[FS_MAX_PATH]; + FILE *fifo = 0; + fileHandle_t handle; + fs_pipe_handle_t *handle_entry; + + if (fs_generate_path_writedir(FS_GetCurrentGameDir(), filename, 0, + FS_ALLOW_DIRECTORIES | FS_CREATE_DIRECTORIES_FOR_FILE, path, sizeof(path))) + { + fifo = Sys_Mkfifo(path); + } + + if (!fifo) + { + Com_Printf(S_COLOR_YELLOW + "WARNING: Could not create new com_pipefile at %s. " + "com_pipefile will not be used.\n", + path); + return 0; + } + + // if(fs_debug->integer) { + // Com_Printf( "FS_FCreateOpenPipeFile: %s\n", ospath ); } + + // Set up handle entry + handle = fs_allocate_handle(FS_HANDLE_PIPE); + handle_entry = (fs_pipe_handle_t *)fs_get_handle_entry(handle); + handle_entry->handle = fifo; + handle_entry->h.debug_path = CopyString(filename); + return handle; +} + +static unsigned int fs_pipe_handle_read(void *buffer, int len, fs_pipe_handle_t *handle_entry) +{ + return fread(buffer, 1, len, handle_entry->handle); +} + +static void fs_pipe_handle_close(fs_pipe_handle_t *handle_entry) { fclose(handle_entry->handle); } + +/* ******************************************************************************** */ +// Common handle operations +/* ******************************************************************************** */ + +void fs_handle_close(fileHandle_t handle) +{ + // Get handle entry + fs_handle_t *handle_entry; + if (!handle) + { + Com_Printf("^1WARNING: fs_handle_close on null handle\n"); + return; + } + handle_entry = fs_get_handle_entry(handle); + if (!handle_entry) + { + Com_Printf("^1WARNING: fs_handle_close on invalid handle\n"); + return; + } + + switch (handle_entry->type) + { + case FS_HANDLE_CACHE_READ: + fs_cache_read_handle_close((fs_cache_read_handle_t *)handle_entry); + break; + case FS_HANDLE_DIRECT_READ: + fs_direct_read_handle_close((fs_direct_read_handle_t *)handle_entry); + break; + case FS_HANDLE_PK3_READ: + fs_pk3_read_handle_close((fs_pk3_read_handle_t *)handle_entry); + break; + case FS_HANDLE_WRITE: + fs_write_handle_close((fs_write_handle_t *)handle_entry); + break; + case FS_HANDLE_PIPE: + fs_pipe_handle_close((fs_pipe_handle_t *)handle_entry); + break; + default: + Com_Error(ERR_DROP, "fs_handle_close invalid handle type"); + } + + Z_Free(handle_entry->debug_path); + fs_free_handle(handle); +} + +void fs_close_all_handles(void) +{ + // Can be used when the whole program is terminating, just to be safe + int i; + for (i = 0; i < MAX_HANDLES; ++i) + { + if (handles[i]) + fs_handle_close(i + 1); + } +} + +static unsigned int fs_handle_read(char *buffer, unsigned int length, fileHandle_t handle) +{ + // Get handle entry + fs_handle_t *handle_entry = fs_get_handle_entry(handle); + if (!handle_entry) + { + Com_Error(ERR_DROP, "fs_handle_read on invalid handle"); + } + + switch (handle_entry->type) + { + case FS_HANDLE_CACHE_READ: + return fs_cache_read_handle_read(buffer, length, (fs_cache_read_handle_t *)handle_entry); + case FS_HANDLE_DIRECT_READ: + return fs_direct_read_handle_read(buffer, length, (fs_direct_read_handle_t *)handle_entry); + case FS_HANDLE_PK3_READ: + return fs_pk3_read_handle_read(buffer, length, (fs_pk3_read_handle_t *)handle_entry); + case FS_HANDLE_PIPE: + return fs_pipe_handle_read(buffer, length, (fs_pipe_handle_t *)handle_entry); + default: + Com_Error(ERR_DROP, "fs_handle_read invalid handle type"); + } + return 0; +} + +static int fs_handle_seek(fileHandle_t handle, int offset, fsOrigin_t origin_mode) +{ + // Get handle entry + fs_handle_t *handle_entry = fs_get_handle_entry(handle); + if (!handle_entry) + { + Com_Error(ERR_DROP, "fs_handle_seek on invalid handle"); + } + + switch (handle_entry->type) + { + case FS_HANDLE_CACHE_READ: + return fs_cache_read_handle_seek((fs_cache_read_handle_t *)handle_entry, offset, origin_mode); + case FS_HANDLE_DIRECT_READ: + return fs_direct_read_handle_seek((fs_direct_read_handle_t *)handle_entry, offset, origin_mode); + case FS_HANDLE_PK3_READ: + return fs_pk3_read_handle_seek((fs_pk3_read_handle_t *)handle_entry, offset, origin_mode); + case FS_HANDLE_WRITE: + return fs_write_handle_seek((fs_write_handle_t *)handle_entry, offset, origin_mode); + default: + Com_Error(ERR_DROP, "fs_handle_seek invalid handle type"); + } + return 1; +} + +static unsigned int fs_handle_ftell(fileHandle_t handle) +{ + // Get handle entry + fs_handle_t *handle_entry = fs_get_handle_entry(handle); + if (!handle_entry) + Com_Error(ERR_DROP, "fs_handle_ftell on invalid handle"); + + switch (handle_entry->type) + { + case FS_HANDLE_CACHE_READ: + return ((fs_cache_read_handle_t *)(handle_entry))->position; + case FS_HANDLE_WRITE: + return fsc_ftell(((fs_write_handle_t *)(handle_entry))->fsc_handle); + default: + Com_Error(ERR_DROP, "fs_handle_ftell invalid handle type"); + } + return 0; +} + +static void fs_handle_set_owner(fileHandle_t handle, fs_handle_owner_t owner) +{ + fs_handle_t *handle_entry = fs_get_handle_entry(handle); + if (!handle_entry) + Com_Error(ERR_DROP, "fs_handle_set_owner on invalid handle"); + handle_entry->owner = owner; +} + +fs_handle_owner_t fs_handle_get_owner(fileHandle_t handle) +{ + fs_handle_t *handle_entry = fs_get_handle_entry(handle); + if (!handle_entry) + return FS_HANDLEOWNER_SYSTEM; + return handle_entry->owner; +} + +static const char *fs_handletype_string(fs_handle_type_t type) +{ + if (type == FS_HANDLE_CACHE_READ) + return "cache read"; + if (type == FS_HANDLE_DIRECT_READ) + return "direct read"; + if (type == FS_HANDLE_PK3_READ) + return "pk3 read"; + if (type == FS_HANDLE_WRITE) + return "write"; + if (type == FS_HANDLE_PIPE) + return "pipe"; + return "unknown"; +} + +static const char *fs_owner_string(fs_handle_owner_t owner) +{ + if (owner == FS_HANDLEOWNER_SYSTEM) + return "system"; + if (owner == FS_HANDLEOWNER_CGAME) + return "cgame"; + if (owner == FS_HANDLEOWNER_UI) + return "ui"; + if (owner == FS_HANDLEOWNER_QAGAME) + return "qagame"; + return "unknown"; +} + +void fs_print_handle_list(void) +{ + int i; + for (i = 0; i < MAX_HANDLES; ++i) + { + if (!handles[i]) + continue; + Com_Printf("********** handle %i **********\n type: %s\n owner: %s\n path: %s\n", i + 1, + fs_handletype_string(handles[i]->type), fs_owner_string(handles[i]->owner), handles[i]->debug_path); + } +} + +void fs_close_owner_handles(fs_handle_owner_t owner) +{ + // Can be called when a VM is shutting down to avoid leaked handles + int i; + for (i = 0; i < MAX_HANDLES; ++i) + { + if (handles[i] && handles[i]->owner == owner) + { + Com_Printf( + "^1*****************\nWARNING: Auto-closing possible leaked handle\n" + "type: %s\nowner: %s\npath: %s\n*****************\n", + fs_handletype_string(handles[i]->type), fs_owner_string(handles[i]->owner), handles[i]->debug_path); + fs_handle_close(i + 1); + } + } +} + +/* ******************************************************************************** */ +// Journal Data File Functions +/* ******************************************************************************** */ + +void fs_write_journal_data(const char *data, unsigned int length) +{ + // Use length 0 to indicate file not found + if (!com_journalDataFile || com_journal->integer != 1) + return; + FS_Write(&length, sizeof(length), com_journalDataFile); + if (length) + FS_Write(data, length, com_journalDataFile); + FS_Flush(com_journalDataFile); +} + +char *fs_read_journal_data(void) +{ + // Returns next piece of data from journal data file, or null if not available + // If data was returned, result needs to be freed by fs_free_data. + unsigned int length; + int r; + char *data; + + if (!com_journalDataFile || com_journal->integer != 2) + return 0; + + r = FS_Read(&length, sizeof(length), com_journalDataFile); + if (r != sizeof(length)) + return 0; + if (!length) + return 0; + + // Obtain buffer from cache or malloc + { + cache_entry_t *cache_entry = cache_allocate(0, length + 1); + if (cache_entry) + { + ++cache_entry->lock_count; + data = CACHE_ENTRY_DATA(cache_entry); + } + else + data = (char *)fsc_malloc(length + 1); + } + + // Attempt to read data + r = FS_Read(data, length, com_journalDataFile); + if (r != length) + Com_Error(ERR_FATAL, "Failed to read data from journal data file"); + data[length] = 0; + return data; +} + +/* ******************************************************************************** */ +// Config File Operations +/* ******************************************************************************** */ + +fileHandle_t fs_open_settings_file_write(const char *filename) +{ + // This is used for writing the primary auto-saved settings file, e.g. q3config.cfg. + // The save directory will be adjusted depending on the fs_mod_settings value. + char path[FS_MAX_PATH]; + const char *mod_dir; + + if (fs_mod_settings->integer) + { + mod_dir = FS_GetCurrentGameDir(); + } + else + { + mod_dir = fs_basegame->string; + } + + if (!fs_generate_path_writedir(mod_dir, filename, FS_CREATE_DIRECTORIES, FS_ALLOW_SPECIAL_CFG, path, sizeof(path))) + return 0; + return fs_write_handle_open(path, qfalse, qfalse); +} + +/* ******************************************************************************** */ +// "Read-back" tracking +/* ******************************************************************************** */ + +// In rare cases, mods may attempt to read files that were just created by the mod/engine. +// This may fail if the file index is not refreshed after the file is created. +// To handle this situation, this module stores a log of files written by the game since +// the last filesystem refresh. If a mod tries to open a file with the same path, a +// filesystem refresh will be performed ahead of the read operation. + +#define MAX_READBACK_TRACKER_ENTRIES 32 +static char *fs_readback_tracker_entries[MAX_READBACK_TRACKER_ENTRIES]; +static int fs_readback_tracker_entry_count = 0; + +static qboolean fs_readback_tracker_process_path(const char *path, qboolean insert) +{ + // Returns qtrue if path exists in registry, qfalse otherwise + // Set "insert" to qtrue to add path to registry + int i; + for (i = 0; i < fs_readback_tracker_entry_count; ++i) + { + if (!Q_stricmp(path, fs_readback_tracker_entries[i])) + return qtrue; + } + if (insert && fs_readback_tracker_entry_count < MAX_READBACK_TRACKER_ENTRIES) + { + fs_readback_tracker_entries[fs_readback_tracker_entry_count++] = CopyString(path); + } + return qfalse; +} + +void fs_readback_tracker_reset(void) +{ + // Resets registry; should be called after filesystem refresh + int i; + for (i = 0; i < fs_readback_tracker_entry_count; ++i) + { + Z_Free(fs_readback_tracker_entries[i]); + } + fs_readback_tracker_entry_count = 0; +} + +/* ******************************************************************************** */ +// FS_FOpenFile Functions +/* ******************************************************************************** */ + +// The FS_FOpenFileByMode family of functions are accessed by both the engine and VM calls, +// and have some peculiar syntax and return values inherited from the original filesystem +// that need to be maintained for compatibility purposes. + +// FS_FOpenFileByMode with write-type mode (FS_WRITE, FS_APPEND, FS_APPEND_SYNC): +// On success, writes handle and returns 0 +// On error, writes null handle and returns -1 + +// FS_FOpenFileByMode with FS_READ and handle pointer set: +// On success, writes handle and returns file size value >= 0 +// On error, writes null handle and returns -1 + +// FS_FOpenFileByMode with FS_READ and null handle pointer (size check mode): +// If file invalid or doesn't exist, returns 0 +// If file exists with size 0, returns 1 +// If file exists with size > 0, returns size + +static int fs_fopenfile_read_handle_open( + const char *filename, fileHandle_t *handle_out, int lookup_flags, qboolean allow_direct_handle) +{ + // Can be called with null handle_out for a size/existance check + // Returns size according to original FS_FOpenFileReadDir conventions + const fsc_file_t *fscfile; + int size = -1; + fileHandle_t handle = 0; + + // Get the file + fscfile = fs_general_lookup(filename, lookup_flags, qfalse); + if (!fscfile) + goto finish; + + // For most size-check cases we can just return the fsc filesize without trying to open the file + if (!handle_out && !(allow_direct_handle && fscfile->sourcetype == FSC_SOURCETYPE_DIRECT)) + { + size = (int)(fscfile->filesize); + goto finish; + } + + // Get the handle and size + if (allow_direct_handle && fscfile->sourcetype == FSC_SOURCETYPE_DIRECT) + { + handle = fs_direct_read_handle_open(fscfile, 0, (unsigned int *)&size); + if (!handle) + size = -1; + } + else if (allow_direct_handle && fscfile->sourcetype == FSC_SOURCETYPE_PK3 && fscfile->filesize > 65536) + { + handle = fs_pk3_read_handle_open(fscfile); + if (handle) + size = (int)(fscfile->filesize); + } + else + { + handle = fs_cache_read_handle_open(fscfile, 0, (unsigned int *)&size); + if (!handle) + size = -1; + } + +finish: + if (handle && size < 0) + { + // This should be very unlikely, but if for some reason we got a handle with an invalid size, + // don't return it because it could cause bugs down the line + fs_handle_close(handle); + handle = 0; + size = -1; + } + + if (handle_out) + { + // Caller wants to keep handle + *handle_out = handle; + } + else + { + // Size check only - modify size as per original FS_FOpenFileReadDir + if (size < 0) + size = 0; + else if (size == 0) + size = 1; + if (handle) + fs_handle_close(handle); + } + + return size; +} + +static fileHandle_t fs_fopenfile_write_handle_open( + const char *mod_dir, const char *path, qboolean append, qboolean sync, int flags) +{ + // Includes directory creation and sanity checks + // Returns handle on success, null on error + char full_path[FS_MAX_PATH]; + + if (!fs_generate_path_writedir(mod_dir, path, FS_CREATE_DIRECTORIES, + FS_ALLOW_DIRECTORIES | FS_CREATE_DIRECTORIES_FOR_FILE | flags, full_path, sizeof(full_path))) + { + if (fs_debug_fileio->integer) + FS_DPrintf("WARNING: Failed to generate write path for %s/%s\n", mod_dir, path); + return 0; + } + + if (mod_dir) + fs_readback_tracker_process_path(path, qtrue); + + return fs_write_handle_open(full_path, append, sync); +} + +static const char *fs_tremulous_write_mod_location(fs_handle_owner_t owner) +{ + // For Tremulous, engine writes should always go to fs_basegame regardless of active mod + // Game module writes can still go to mod directory if set + if (owner == FS_HANDLEOWNER_SYSTEM) + return fs_basegame->string; + return FS_GetCurrentGameDir(); +} + +static bool FS_IsExt(const char *filename, const char *ext, int namelen) +{ + int extlen = strlen(ext); + if (extlen > namelen) + return false; + filename += namelen - extlen; + return !Q_stricmp(filename, ext); +} + +static int FS_FOpenFileByModeGeneral(const char *qpath, fileHandle_t *f, fsMode_t mode, fs_handle_owner_t owner) +{ + // Can be called with a null filehandle pointer in read mode for a size/existance check + int size = 0; + fileHandle_t handle = 0; + + if (!qpath) + { + Com_Error(ERR_DROP, "FS_FOpenFileByMode: null path"); + } + if (!f && mode != FS_READ) + { + Com_Error(ERR_DROP, "FS_FOpenFileByMode: null handle pointer with non-read mode"); + } + + if (mode == FS_READ) + { + int lookup_flags = 0; + + // Tremulous: Allow specific extensions to be read outside pk3s when connected to a pure server + int len = fsc_strlen(qpath); + if (FS_IsExt(qpath, ".cfg", len) || FS_IsExt(qpath, ".dat", len) || FS_IsExt(qpath, ".menu", len)) + lookup_flags |= LOOKUPFLAG_PURE_ALLOW_DIRECT_SOURCE; + + if (owner != FS_HANDLEOWNER_SYSTEM) + { + if (fs_readback_tracker_process_path(qpath, qfalse)) + { + // If file was potentially just written, run filesystem refresh to make sure it is registered + if (fs_debug_fileio->integer) + { + FS_DPrintf("Running filesystem refresh due to recently written file %s\n", qpath); + } + fs_refresh(qtrue); + } + + if (owner == FS_HANDLEOWNER_QAGAME) + { + // Ignore pure list for server VM. This prevents the server mod from being affected + // by the pure list when running a local game with sv_pure enabled. + lookup_flags |= LOOKUPFLAG_IGNORE_PURE_LIST; + } + + // Use read with direct handle support option, to ensure recently/actively written files + // on disk are opened properly, and to optimize for pk3 read operations that read only the + // beginning of the file (e.g. UI Enhanced mod doing bulk bsp reads on startup) + if (!handle) + size = fs_fopenfile_read_handle_open(qpath, f ? &handle : 0, lookup_flags, qtrue); + } + + // Engine reads don't do anything fancy so just use the basic method + else + size = fs_fopenfile_read_handle_open(qpath, f ? &handle : 0, lookup_flags, qfalse); + } + else if (mode == FS_WRITE) + { + handle = fs_fopenfile_write_handle_open(fs_tremulous_write_mod_location(owner), qpath, qfalse, qfalse, 0); + } + else if (mode == FS_APPEND_SYNC) + { + handle = fs_fopenfile_write_handle_open(fs_tremulous_write_mod_location(owner), qpath, qtrue, qtrue, 0); + } + else if (mode == FS_APPEND) + { + handle = fs_fopenfile_write_handle_open(fs_tremulous_write_mod_location(owner), qpath, qtrue, qfalse, 0); + } + else + { + Com_Error(ERR_DROP, "FS_FOpenFileByMode: bad mode"); + } + + if (f) + { + // Caller wants to keep the handle + *f = handle; + if (handle) + fs_handle_set_owner(handle, owner); + else + size = -1; + } + + return size; +} + +static const char *fs_mode_string(fsMode_t mode) +{ + if (mode == FS_READ) + return "read"; + if (mode == FS_WRITE) + return "write"; + if (mode == FS_APPEND) + return "append"; + if (mode == FS_APPEND_SYNC) + return "append-sync"; + return "unknown"; +} + +static int FS_FOpenFileByModeLogged( + const char *qpath, fileHandle_t *f, fsMode_t mode, fs_handle_owner_t owner, const char *calling_function) +{ + int result; + + if (fs_debug_fileio->integer) + { + FS_DPrintf("********** file handle open **********\n"); + fs_debug_indent_start(); + FS_DPrintf("origin: %s\n", calling_function); + FS_DPrintf("path: %s\n", qpath); + if (mode == FS_READ && !f) + { + FS_DPrintf("mode: read (size check)\n"); + } + else + { + FS_DPrintf("mode: %s\n", fs_mode_string(mode)); + } + FS_DPrintf("owner: %s\n", fs_owner_string(owner)); + } + + result = FS_FOpenFileByModeGeneral(qpath, f, mode, owner); + + if (fs_debug_fileio->integer) + { + FS_DPrintf("result: return value %i (handle %i)\n", result, f ? *f : 0); + fs_debug_indent_stop(); + } + + return result; +} + +long FS_FOpenFileRead(const char *filename, fileHandle_t *file, qboolean uniqueFILE) +{ + FSC_ASSERT(filename); + return FS_FOpenFileByModeLogged(filename, file, FS_READ, FS_HANDLEOWNER_SYSTEM, "FS_FOpenFileRead"); +} + +fileHandle_t FS_FOpenFileWrite(const char *filename) +{ + fileHandle_t handle = 0; + FSC_ASSERT(filename); + FS_FOpenFileByModeLogged(filename, &handle, FS_WRITE, FS_HANDLEOWNER_SYSTEM, "FS_FOpenFileWrite"); + return handle; +} + +fileHandle_t FS_FOpenFileAppend(const char *filename) +{ + fileHandle_t handle = 0; + FSC_ASSERT(filename); + FS_FOpenFileByModeLogged(filename, &handle, FS_APPEND, FS_HANDLEOWNER_SYSTEM, "FS_FOpenFileAppend"); + return handle; +} + +int FS_FOpenFileByModeOwner(const char *qpath, fileHandle_t *f, fsMode_t mode, fs_handle_owner_t owner) +{ + return FS_FOpenFileByModeLogged(qpath, f, mode, owner, "FS_FOpenFileByModeOwner"); +} + +int FS_FOpenFileByMode(const char *qpath, fileHandle_t *f, fsMode_t mode) +{ + FSC_ASSERT(qpath); + return FS_FOpenFileByModeLogged(qpath, f, mode, FS_HANDLEOWNER_SYSTEM, "FS_FOpenFileByMode"); +} + +/* ******************************************************************************** */ +// Misc Handle Operations +/* ******************************************************************************** */ + +long FS_SV_FOpenFileRead(const char *filename, fileHandle_t *fp) +{ + int i; + char path[FS_MAX_PATH]; + unsigned int size = -1; + FSC_ASSERT(filename); + FSC_ASSERT(fp); + *fp = 0; + + if (fs_debug_fileio->integer) + { + FS_DPrintf("********** SV file read **********\n"); + fs_debug_indent_start(); + FS_DPrintf("path: %s\n", filename); + } + + for (i = 0; i < FS_MAX_SOURCEDIRS; ++i) + { + if (fs_generate_path_sourcedir(i, filename, 0, FS_ALLOW_DIRECTORIES, 0, path, sizeof(path))) + { + *fp = fs_cache_read_handle_open(0, path, &size); + if (*fp) + break; + } + } + if (!*fp) + size = -1; + + if (fs_debug_fileio->integer) + { + FS_DPrintf("result: return value %i (handle %i)\n", size, *fp); + fs_debug_indent_stop(); + } + + return size; +} + +fileHandle_t FS_SV_FOpenFileWrite(const char *filename) +{ + FSC_ASSERT(filename); + return fs_fopenfile_write_handle_open(0, filename, qfalse, qfalse, 0); +} + +void FS_FCloseFile(fileHandle_t f) +{ + if (!f) + { + Com_DPrintf("FS_FCloseFile on null handle\n"); + return; + } + fs_handle_close(f); +} + +int FS_Read(void *buffer, int len, fileHandle_t f) +{ + FSC_ASSERT(buffer); + return fs_handle_read((char *)buffer, len, f); +} + +int FS_Read2(void *buffer, int len, fileHandle_t f) +{ + // This seems pretty much identical to FS_Read in the original filesystem as well + FSC_ASSERT(buffer); + return FS_Read(buffer, len, f); +} + +int FS_Write(const void *buffer, int len, fileHandle_t h) +{ + FSC_ASSERT(buffer); + fs_write_handle_write(h, (const char *)buffer, len); + return len; +} + +int FS_Seek(fileHandle_t f, long offset, int origin) { return fs_handle_seek(f, offset, (fsOrigin_t)origin); } + +int FS_FTell(fileHandle_t f) { return fs_handle_ftell(f); } + +void FS_Flush(fileHandle_t f) { fs_write_handle_flush(f, qfalse); } + +void FS_ForceFlush(fileHandle_t f) { fs_write_handle_flush(f, qtrue); } + +void FS_SV_Rename(const char *from, const char *to, qboolean safe) +{ + char source_path[FS_MAX_PATH]; + char target_path[FS_MAX_PATH]; + + if (!fs_generate_path_writedir(from, 0, FS_ALLOW_DIRECTORIES, 0, source_path, sizeof(source_path))) + return; + if (!fs_generate_path_writedir( + to, 0, FS_ALLOW_DIRECTORIES | FS_CREATE_DIRECTORIES_FOR_FILE, 0, target_path, sizeof(target_path))) + return; + fs_rename_file(source_path, target_path); +} + +/* ******************************************************************************** */ +// Misc Data Operations +/* ******************************************************************************** */ + +long FS_ReadFile(const char *qpath, void **buffer) +{ + // Returns -1 and nulls buffer on error. Returns size and sets buffer on success. + // On success result buffer must be freed with FS_FreeFile. + // Can be called with null buffer for size check. + const fsc_file_t *file; + unsigned int len; + FSC_ASSERT(qpath); + + file = fs_general_lookup(qpath, 0, qfalse); + + if (!file) + { + // File not found + if (buffer) + *buffer = 0; + return -1; + } + + if (!buffer) + { + // Size check + return (long)file->filesize; + } + + *buffer = fs_read_data(file, 0, &len, "FS_ReadFile"); + return (long)len; +} + +void FS_FreeFile(void *buffer) +{ + if (!buffer) + Com_Error(ERR_FATAL, "FS_FreeFile( NULL )"); + fs_free_data((char *)buffer); +} + +void FS_WriteFile(const char *qpath, const void *buffer, int size) +{ + // Copy from original filesystem + fileHandle_t f; + + if (!qpath || !buffer) + { + Com_Error(ERR_FATAL, "FS_WriteFile: NULL parameter"); + } + + f = FS_FOpenFileWrite(qpath); + if (!f) + { + Com_Printf("Failed to open %s\n", qpath); + return; + } + + FS_Write(buffer, size, f); + + FS_FCloseFile(f); +} + +#endif // NEW_FILESYSTEM diff --git a/src/filesystem/fs_filelist.cpp b/src/filesystem/fs_filelist.cpp new file mode 100644 index 000000000..86a634221 --- /dev/null +++ b/src/filesystem/fs_filelist.cpp @@ -0,0 +1,908 @@ +/* +=========================================================================== +Copyright (C) 1999-2005 Id Software, Inc. +Copyright (C) 2017 Noah Metzger (chomenor@gmail.com) + +This file is part of Quake III Arena source code. + +Quake III Arena source code is free software; you can redistribute it +and/or modify it under the terms of the GNU General Public License as +published by the Free Software Foundation; either version 2 of the License, +or (at your option) any later version. + +Quake III Arena source code is distributed in the hope that it will be +useful, but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with Quake III Arena source code; if not, write to the Free Software +Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA +=========================================================================== +*/ + +#ifdef NEW_FILESYSTEM +#include "fslocal.h" + +// Limit max files returned by searches to avoid Z_Malloc overflows +#define MAX_FOUND_FILES 32768 + +typedef struct { + const char *extension; + const char *filter; + int flags; +} filelist_query_t; + +typedef struct { + const char *extension; + const char *filter; + int flags; + + int crop_length; + fsc_stack_t temp_stack; + + // Depth is max number of slash-separated sections allowed in output + // i.e. depth=0 suppresses any output, depth=1 allows "file" and "dir1/", depth=2 allows "dir1/file" and + // "dir1/dir2/" Direct depth applies to files on disk (outside of pk3s), general applies to files in pk3s + int general_file_depth; + int general_directory_depth; + int direct_file_depth; + int direct_directory_depth; +} filelist_work_t; + +// Treat pk3dirs the same as pk3s here +#define DIRECT_NON_PK3DIR(file) (file->sourcetype == FSC_SOURCETYPE_DIRECT && !((fsc_file_direct_t *)file)->pk3dir_ptr) + +/* ******************************************************************************** */ +// Sort key handling +/* ******************************************************************************** */ + +typedef struct { + int length; + char key[]; +} file_list_sort_key_t; + +static file_list_sort_key_t *generate_sort_key(const fsc_file_t *file, fsc_stack_t *stack, const filelist_work_t *flw) +{ + char buffer[1024]; + fsc_stream_t stream = {buffer, 0, sizeof(buffer), 0}; + file_list_sort_key_t *key; + + fs_generate_core_sort_key(file, &stream, (flw->flags & FLISTFLAG_IGNORE_PURE_LIST) ? qfalse : qtrue); + + key = + (file_list_sort_key_t *)FSC_STACK_RETRIEVE(stack, fsc_stack_allocate(stack, sizeof(*key) + stream.position), 0); + key->length = stream.position; + fsc_memcpy(key->key, stream.data, stream.position); + return key; +} + +static int compare_sort_keys(file_list_sort_key_t *key1, file_list_sort_key_t *key2) +{ + return fsc_memcmp(key2->key, key1->key, key1->length < key2->length ? key1->length : key2->length); +} + +/* ******************************************************************************** */ +// String processing functions +/* ******************************************************************************** */ + +static qboolean fs_pattern_match(const char *string, const char *pattern, qboolean initial_wildcard) +{ + // Returns qtrue if string matches pattern containing '*' and '?' wildcards + // Set initial_wildcard to qtrue to process pattern as if the first character was an asterisk + while (1) + { + if (*pattern == '*' || initial_wildcard) + { + // Skip asterisks; auto match if no pattern remaining + char lwr, upr; + while (*pattern == '*') + ++pattern; + if (!*pattern) + return qtrue; + + // Get 'lwr' and 'upr' versions of next char in pattern for fast comparison + lwr = tolower(*pattern); + upr = toupper(*pattern); + + // Read string looking for match with remaining pattern + while (*string) + { + if (*string == lwr || *string == upr || *pattern == '?') + { + if (fs_pattern_match(string + 1, pattern + 1, qfalse)) + return qtrue; + } + ++string; + } + + // Leftover pattern with no match + return qfalse; + } + + // Check for end of string cases + if (!*pattern) + { + if (!*string) + return qtrue; + return qfalse; + } + if (!*string) + return qfalse; + + // Check for character discrepancy + if (*pattern != *string && *pattern != '?' && tolower(*pattern) != tolower(*string)) + return qfalse; + + // Advance strings + ++pattern; + ++string; + } +} + +static void sanitize_path_separators(const char *source, char *target, int target_size) +{ + // Sanitize os-specific path separator content (like ./ or //) out of the path string + int target_index = 0; + qboolean slash_mode = qfalse; + + while (*source) + { + char current = *(source++); + if (current == '\\') + current = '/'; + + // Defer writing slashes until a valid character is encountered + if (current == '/') + { + slash_mode = qtrue; + continue; + } + + // Ignore periods that are followed by slashes or end of string + if (current == '.' && (*source == '/' || *source == '\\' || *source == '\0')) + continue; + + // Write out deferred slashes unless at the beginning of path + if (slash_mode) + { + slash_mode = qfalse; + if (target_index) + { + if (target_index + 2 >= target_size) + break; + target[target_index++] = '/'; + } + } + + // Write character + if (target_index + 1 >= target_size) + break; + target[target_index++] = current; + } + + target[target_index] = 0; +} + +static qboolean strip_trailing_slash(const char *source, char *target, int target_size) +{ + // Removes single trailing slash from path; returns qtrue if found + int length; + Q_strncpyz(target, source, target_size); + length = strlen(target); + if (length && (target[length - 1] == '/' || target[length - 1] == '\\')) + { + target[length - 1] = 0; + return qtrue; + } + return qfalse; +} + +/* ******************************************************************************** */ +// File list generation +/* ******************************************************************************** */ + +typedef struct { + fs_hashtable_entry_t hte; + char *string; + const fsc_file_t *file; + qboolean directory; + file_list_sort_key_t *file_sort_key; +} temp_file_set_entry_t; + +static char *allocate_string(const char *string) +{ + int length = strlen(string); + char *copy = (char *)Z_Malloc(length + 1); + Com_Memcpy(copy, string, length); + copy[length] = 0; + return copy; +} + +static void temp_file_set_insert(fs_hashtable_t *ht, const fsc_file_t *file, const char *path, qboolean directory, + file_list_sort_key_t **sort_key_ptr, filelist_work_t *flw) +{ + // Loads a path/file combination into the file set, displacing lower precedence entries if needed + unsigned int hash = fsc_string_hash(path, 0); + fs_hashtable_iterator_t it = fs_hashtable_iterate(ht, hash, qfalse); + temp_file_set_entry_t *entry; + + if (ht->element_count >= MAX_FOUND_FILES) + return; + + // Generate sort key if one was not already created for this file + if (!*sort_key_ptr) + *sort_key_ptr = generate_sort_key(file, &flw->temp_stack, flw); + + while ((entry = (temp_file_set_entry_t *)fs_hashtable_next(&it))) + { + // Check if file is match + if (strcmp(path, entry->string)) + continue; + + // Matching file found - replace it if new file is higher precedence + if (compare_sort_keys(*sort_key_ptr, entry->file_sort_key) < 0) + { + entry->file = file; + entry->directory = directory; + entry->file_sort_key = *sort_key_ptr; + } + return; + } + + // No matching file - create new entry + entry = (temp_file_set_entry_t *)Z_Malloc(sizeof(*entry)); + entry->file = file; + entry->directory = directory; + entry->file_sort_key = *sort_key_ptr; + entry->string = allocate_string(path); + fs_hashtable_insert(ht, &entry->hte, hash); +} + +static qboolean check_path_enabled(fsc_stream_t *stream, const filelist_work_t *flw) +{ + // Returns qtrue if path stored in stream matches filter/extension criteria, qfalse otherwise + if (!stream->position) + return qfalse; + if (flw->extension) + { + if (!fs_pattern_match(stream->data, flw->extension, qtrue)) + return qfalse; + } + if (flw->filter) + { + if (!Com_FilterPath((char *)flw->filter, stream->data, qfalse)) + return qfalse; + } + return qtrue; +} + +static qboolean check_file_enabled(const fsc_file_t *file, const filelist_work_t *flw) +{ + // Returns qtrue if file is valid to use, qfalse otherwise + int disabled_checks = FD_CHECK_FILE_ENABLED | FD_CHECK_LIST_INACTIVE_MODS | FD_CHECK_LIST_AUXILIARY_SOURCEDIR; + if (!(flw->flags & FLISTFLAG_IGNORE_PURE_LIST) && + !((flw->flags & FLISTFLAG_PURE_ALLOW_DIRECT_SOURCE) && file->sourcetype == FSC_SOURCETYPE_DIRECT)) + { + disabled_checks |= FD_CHECK_PURE_LIST; + } + if (fs_file_disabled(file, disabled_checks)) + return qfalse; + if ((flw->flags & FLISTFLAG_IGNORE_TAPAK0) && file->sourcetype == FSC_SOURCETYPE_PK3 && + fsc_get_base_file(file, &fs)->pk3_hash == 2430342401u) + return qfalse; + return qtrue; +} + +static void cut_stream(const fsc_stream_t *source, fsc_stream_t *target, int start_pos, int end_pos) +{ + target->position = 0; + if (start_pos < end_pos) + fsc_write_stream_data(target, source->data + start_pos, end_pos - start_pos); + fsc_stream_append_string(target, ""); +} + +static void temp_file_set_populate(const fsc_directory_t *base, fs_hashtable_t *output, filelist_work_t *flw) +{ + char path_buffer[FS_FILE_BUFFER_SIZE]; + fsc_stream_t path_stream = {path_buffer, 0, sizeof(path_buffer), 0}; + char string_buffer[FS_FILE_BUFFER_SIZE]; + fsc_stream_t string_stream = {string_buffer, 0, sizeof(string_buffer), 0}; + fsc_file_t *file; + fsc_directory_t *directory; + + file = (fsc_file_t *)STACKPTRN(base->sub_file); + while (file) + { + if (check_file_enabled(file, flw)) + { + int directory_depth = DIRECT_NON_PK3DIR(file) ? flw->direct_directory_depth : flw->general_directory_depth; + int file_depth = DIRECT_NON_PK3DIR(file) ? flw->direct_file_depth : flw->general_file_depth; + int i, j; + file_list_sort_key_t *sort_key = 0; + int depth = 0; + + // Generate file and directory strings for each file, and call temp_file_set_insert + // For example, a file with post-crop_length string "abc/def/temp.txt" will generate: + // - file string "abc/def/temp.txt" if file depth >= 3 + // - if the file is in a pk3, "abc/" if dir depth >= 1, and "abc/def/" if dir depth >= 2 + // - if file is on disk, ["abc", ".", ".."] if dir depth >= 1, ["abc/def", "abc/.", "abc/.."] + // if dir depth >= 2, and ["abc/def/.", "abc/def/.."] if dir depth >= 3 + + path_stream.position = 0; + fs_file_to_stream(file, &path_stream, qfalse, qfalse, qfalse, qfalse); + for (i = flw->crop_length; i < path_stream.position; ++i) + { + if (path_stream.data[i] == '/') + { + depth++; + if (depth <= directory_depth) + { + // Process directory + cut_stream(&path_stream, &string_stream, flw->crop_length, i); + // Include trailing slash unless directory is from disk, as per original filesystem behavior + if (!DIRECT_NON_PK3DIR(file)) + fsc_stream_append_string(&string_stream, "/"); + if (check_path_enabled(&string_stream, flw)) + { + temp_file_set_insert(output, file, string_stream.data, qtrue, &sort_key, flw); + } + } + } + + // Generate "." and ".." entries for directories from disk + if (DIRECT_NON_PK3DIR(file) && (i == flw->crop_length || path_stream.data[i] == '/') && + depth < directory_depth) + { + cut_stream(&path_stream, &string_stream, flw->crop_length, i); + if (i != flw->crop_length) + fsc_stream_append_string(&string_stream, "/"); + for (j = 0; j < 2; ++j) + { + fsc_stream_append_string(&string_stream, "."); + if (check_path_enabled(&string_stream, flw)) + { + temp_file_set_insert(output, file, string_stream.data, qtrue, &sort_key, flw); + } + } + } + } + + if (depth < file_depth) + { + // Process file + cut_stream(&path_stream, &string_stream, flw->crop_length, path_stream.position); + if (check_path_enabled(&string_stream, flw)) + { + temp_file_set_insert(output, file, string_stream.data, qfalse, &sort_key, flw); + } + } + } + + file = (fsc_file_t *)STACKPTRN(file->next_in_directory); + } + + // Process subdirectories + directory = (fsc_directory_t *)STACKPTRN(base->sub_directory); + while (directory) + { + temp_file_set_populate(directory, output, flw); + directory = (fsc_directory_t *)STACKPTRN(directory->peer_directory); + } +} + +static int temp_file_list_compare_string(const temp_file_set_entry_t *element1, const temp_file_set_entry_t *element2) +{ + char buffer1[FS_FILE_BUFFER_SIZE]; + char buffer2[FS_FILE_BUFFER_SIZE]; + fsc_stream_t stream1 = {buffer1, 0, sizeof(buffer1), qfalse}; + fsc_stream_t stream2 = {buffer2, 0, sizeof(buffer2), qfalse}; + // Use shorter-path-first mode for sorting directories, as it is generally better and more consistent + // with original filesystem behavior + fs_write_sort_string(element1->string, &stream1, element1->directory ? qtrue : qfalse); + fs_write_sort_string(element2->string, &stream2, element2->directory ? qtrue : qfalse); + return fsc_memcmp( + stream2.data, stream1.data, stream1.position < stream2.position ? stream1.position : stream2.position); +} + +static int temp_file_list_compare_element(const temp_file_set_entry_t *element1, const temp_file_set_entry_t *element2) +{ + if (element1->file != element2->file) + { + int sort_result = compare_sort_keys(element1->file_sort_key, element2->file_sort_key); + if (sort_result) + return sort_result; + } + return temp_file_list_compare_string(element1, element2); +} + +static int temp_file_list_qsort(const void *element1, const void *element2) +{ + return temp_file_list_compare_element( + *(const temp_file_set_entry_t **)element1, *(const temp_file_set_entry_t **)element2); +} + +static char **temp_file_set_to_file_list(fs_hashtable_t *file_set, int *numfiles_out, filelist_work_t *flw) +{ + int i; + fs_hashtable_iterator_t it; + temp_file_set_entry_t *entry; + int position = 0; + char **output = (char **)Z_Malloc(sizeof(*output) * (file_set->element_count + 1)); + temp_file_set_entry_t **temp_list = + (temp_file_set_entry_t **)Z_Malloc(sizeof(*temp_list) * file_set->element_count); + + // Transfer entries from file set hashtable to temporary list + it = fs_hashtable_iterate(file_set, 0, qtrue); + while ((entry = (temp_file_set_entry_t *)fs_hashtable_next(&it))) + { + if (position >= file_set->element_count) + { + Com_Error(ERR_FATAL, "fs_filelist.c->temp_file_set_to_file_list element_count overflow"); + } + temp_list[position++] = entry; + } + if (position != file_set->element_count) + { + Com_Error(ERR_FATAL, "fs_filelist.c->temp_file_set_to_file_list element_count underflow"); + } + + // Sort the list + qsort(temp_list, file_set->element_count, sizeof(*temp_list), temp_file_list_qsort); + + // Transfer strings from list to output array + for (i = 0; i < file_set->element_count; ++i) + { + output[i] = temp_list[i]->string; + } + output[i] = 0; + + if (numfiles_out) + *numfiles_out = file_set->element_count; + Z_Free(temp_list); + return output; +} + +/* ******************************************************************************** */ +// Main file list function (list_files) +/* ******************************************************************************** */ + +static fsc_directory_t *get_start_directory(const char *path) +{ + // Path can be null to start at base directory + fsc_hashtable_iterator_t hti; + fsc_stackptr_t directory_ptr; + fsc_directory_t *directory = 0; + + // Look for directory entry + fsc_hashtable_open(&fs.directories, path ? fsc_string_hash(path, 0) : 0, &hti); + while ((directory_ptr = fsc_hashtable_next(&hti))) + { + directory = (fsc_directory_t *)STACKPTR(directory_ptr); + if (path) + { + if (!Q_stricmp((const char *)STACKPTR(directory->qp_dir_ptr), path)) + break; + } + else + { + if (directory->qp_dir_ptr == 0) + break; + } + } + + if (!directory_ptr) + return 0; + return directory; +} + +static void filelist_debug_print_flags(int flags) +{ + if (flags) + { + char buffer[256]; + fsc_stream_t stream = {buffer, 0, sizeof(buffer), 0}; + const char *flag_strings[3] = {0}; + + flag_strings[0] = (flags & FLISTFLAG_IGNORE_TAPAK0) ? "ignore_tapak0" : 0; + flag_strings[1] = (flags & FLISTFLAG_IGNORE_PURE_LIST) ? "ignore_pure_list" : 0; + flag_strings[2] = (flags & FLISTFLAG_PURE_ALLOW_DIRECT_SOURCE) ? "pure_allow_direct_source" : 0; + fs_comma_separated_list(flag_strings, ARRAY_LEN(flag_strings), &stream); + + FS_DPrintf("flags: %i (%s)\n", flags, buffer); + } + else + { + FS_DPrintf("flags: \n"); + } +} + +static char **list_files(const char *path, int *numfiles_out, filelist_query_t *query) +{ + char path_buffer1[FSC_MAX_QPATH]; + char path_buffer2[FSC_MAX_QPATH]; + fsc_directory_t *start_directory = 0; + fs_hashtable_t temp_file_set; + filelist_work_t flw; + char **result; + int start_time = 0; + int special_depth = 0; // Account for certain depth-increasing quirks in original filesystem + + if (fs_debug_filelist->integer) + { + start_time = Sys_Milliseconds(); + FS_DPrintf("********** file list query **********\n"); + fs_debug_indent_start(); + FS_DPrintf("path: %s\n", path); + FS_DPrintf("extension: %s\n", query->extension); + FS_DPrintf("filter: %s\n", query->filter); + filelist_debug_print_flags(query->flags); + } + + // Initialize temp structures + Com_Memset(&flw, 0, sizeof(flw)); + flw.extension = query->extension; + flw.filter = query->filter; + flw.flags = query->flags; + fsc_stack_initialize(&flw.temp_stack); + fs_hashtable_initialize(&temp_file_set, MAX_FOUND_FILES); + + // Determine start directory + if (path) + { + if (strip_trailing_slash(path, path_buffer1, sizeof(path_buffer1))) + { + ++special_depth; + } + sanitize_path_separators(path_buffer1, path_buffer2, sizeof(path_buffer2)); + if (*path_buffer2) + { + start_directory = get_start_directory(path_buffer2); + } + else + { + start_directory = get_start_directory(0); + ++special_depth; + } + } + + if (start_directory) + { + // Determine depths + int extension_length; + if (flw.filter) + { + // Unlimited depth in filter mode + flw.general_file_depth = flw.general_directory_depth = 256; + flw.direct_file_depth = flw.direct_directory_depth = 256; + } + else if (!Q_stricmp(flw.extension, "/")) + { + // This extension is handled specially by the original filesystem (via Sys_ListFiles) + // Do a directory-only query, but skip the extension check because directories in this + // mode can be generated without the trailing slash + flw.general_directory_depth = 1 + special_depth; + flw.direct_directory_depth = 1; + flw.extension = 0; + } + else + { + // Roughly emulate original filesystem depth behavior + flw.general_file_depth = 2 + special_depth; + flw.general_directory_depth = 1 + special_depth; + flw.direct_file_depth = 1; + } + + // Optimization to skip processing path types blocked by extension anyway + extension_length = flw.extension ? strlen(flw.extension) : 0; + if (extension_length) + { + if (flw.extension[extension_length - 1] == '/') + { + flw.general_file_depth = flw.direct_file_depth = 0; + } + else if (flw.extension[extension_length - 1] != '?' && flw.extension[extension_length - 1] != '*') + { + flw.general_directory_depth = flw.direct_directory_depth = 0; + } + } + + // Disable non-direct files when emulating OS-specific behavior that would restrict + // output to direct files on original filesystem + // NOTE: Consider restricting general depths to match direct depths in these cases instead of + // disabling them entirely? + if (Q_stricmp(path_buffer1, path_buffer2)) + { + if (fs_debug_filelist->integer) + FS_DPrintf( + "NOTE: Restricting to direct files only due to OS-specific" + " path separator conversion: original(%s) converted(%s)\n", + path_buffer1, path_buffer2); + flw.general_file_depth = flw.general_directory_depth = 0; + } + if (flw.extension && (strchr(flw.extension, '*') || strchr(flw.extension, '?'))) + { + if (fs_debug_filelist->integer) + FS_DPrintf( + "NOTE: Restricting to direct files only due to OS-specific" + " extension wildcards\n"); + flw.general_file_depth = flw.general_directory_depth = 0; + } + + // Debug print depths + if (fs_debug_filelist->integer) + { + FS_DPrintf("depths: gf(%i) gd(%i) df(%i) dd(%i)\n", flw.general_file_depth, flw.general_directory_depth, + flw.direct_file_depth, flw.direct_directory_depth); + } + + // Determine prefix length + if (!flw.filter && start_directory->qp_dir_ptr) + { + flw.crop_length = strlen((const char *)STACKPTR(start_directory->qp_dir_ptr)) + 1; + } + + // Populate file set + temp_file_set_populate(start_directory, &temp_file_set, &flw); + } + else if (fs_debug_filelist->integer) + FS_DPrintf("NOTE: Failed to match start directory.\n"); + + // Generate file list + result = temp_file_set_to_file_list(&temp_file_set, numfiles_out, &flw); + + if (fs_debug_filelist->integer) + { + FS_DPrintf("result: %i elements\n", temp_file_set.element_count); + FS_DPrintf("temp stack usage: %u\n", fsc_stack_get_export_size(&flw.temp_stack)); + FS_DPrintf("time: %i\n", Sys_Milliseconds() - start_time); + fs_debug_indent_stop(); + } + + fs_hashtable_free(&temp_file_set, 0); + fsc_stack_free(&flw.temp_stack); + return result; +} + +void FS_FreeFileList(char **list) +{ + int i; + if (!list) + return; + + for (i = 0; list[i]; i++) + { + Z_Free(list[i]); + } + + Z_Free(list); +} + +/* ******************************************************************************** */ +// Mod directory listing (FS_GetModList) +/* ******************************************************************************** */ + +#define MAX_MOD_DIRS 128 + +typedef struct { + char *mod_dirs[MAX_MOD_DIRS]; + int count; +} mod_dir_list_t; + +static void add_mod_dir_to_list(mod_dir_list_t *list, const char *mod_dir) +{ + int i; + if (list->count >= MAX_MOD_DIRS) + return; + for (i = 0; i < list->count; ++i) + if (!Q_stricmp(list->mod_dirs[i], mod_dir)) + return; + list->mod_dirs[list->count++] = CopyString(mod_dir); +} + +static void generate_mod_dir_list2(fsc_directory_t *base, mod_dir_list_t *list) +{ + fsc_file_t *file; + fsc_directory_t *directory; + const char *last_mod_dir = 0; + + file = (fsc_file_t *)STACKPTRN(base->sub_file); + while (file) + { + if (file->sourcetype == FSC_SOURCETYPE_DIRECT && fsc_is_file_enabled(file, &fs)) + { + const char *mod_dir = fsc_get_mod_dir(file, &fs); + if (mod_dir != last_mod_dir) + { + add_mod_dir_to_list(list, fsc_get_mod_dir(file, &fs)); + last_mod_dir = mod_dir; + } + } + + file = (fsc_file_t *)STACKPTRN(file->next_in_directory); + } + + directory = (fsc_directory_t *)STACKPTRN(base->sub_directory); + while (directory) + { + generate_mod_dir_list2(directory, list); + directory = (fsc_directory_t *)STACKPTRN(directory->peer_directory); + } +} + +static int mod_list_qsort(const void *element1, const void *element2) +{ + return Q_stricmp(*(const char **)element1, *(const char **)element2); +} + +static void generate_mod_dir_list(mod_dir_list_t *list) +{ + list->count = 0; + generate_mod_dir_list2(get_start_directory(0), list); + qsort(list->mod_dirs, list->count, sizeof(*list->mod_dirs), mod_list_qsort); +} + +static void mod_list_free(mod_dir_list_t *list) +{ + int i; + for (i = 0; i < list->count; ++i) + Z_Free(list->mod_dirs[i]); +} + +static int FS_GetModList(char *listbuf, int bufsize) +{ + int i; + int nTotal = 0; // Amount of buffer used so far + int nMods = 0; // Number of mods + mod_dir_list_t list; + char description[49]; + int mod_name_length; + int description_length; + + generate_mod_dir_list(&list); + + for (i = 0; i < list.count; ++i) + { + char *mod_name = list.mod_dirs[i]; + if (!Q_stricmp(mod_name, fs_basegame->string)) + continue; + if (!Q_stricmp(mod_name, "basemod")) + continue; + + FS_GetModDescription(mod_name, description, sizeof(description)); + mod_name_length = strlen(mod_name) + 1; + description_length = strlen(description) + 1; + + if (nTotal + mod_name_length + description_length < bufsize) + { + strcpy(listbuf, mod_name); + listbuf += mod_name_length; + strcpy(listbuf, description); + listbuf += description_length; + nTotal += mod_name_length + description_length; + ++nMods; + } + } + + mod_list_free(&list); + return nMods; +} + +/* ******************************************************************************** */ +// External file list functions +/* ******************************************************************************** */ + +char **FS_FlagListFilteredFiles( + const char *path, const char *extension, const char *filter, int *numfiles_out, int flags) +{ + // path, extension, filter, and numfiles_out may be null + filelist_query_t query = {extension, filter, flags}; + return list_files(path, numfiles_out, &query); +} + +char **FS_ListFiles(const char *path, const char *extension, int *numfiles) +{ + // path, extension, and numfiles may be null + return FS_FlagListFilteredFiles(path, extension, NULL, numfiles, 0); +} + +int FS_GetFileList(const char *path, const char *extension, char *listbuf, int bufsize) +{ + // path and extension may be null + int i; + int flags = 0; + int nFiles = 0; + int nTotal = 0; + int nLen; + char **pFiles = NULL; + FSC_ASSERT(listbuf); + + *listbuf = 0; + + if (!Q_stricmp(path, "$modlist")) + { + return FS_GetModList(listbuf, bufsize); + } + + if (!Q_stricmp(path, "demos")) + { + // Check for new demos before displaying the UI demo menu + fs_refresh_auto(); + } + + /* + if (!Q_stricmp(path, "models/players") && extension && !Q_stricmp(extension, "/") && + Q_stricmp(current_mod_dir, BASETA)) + { + // Special case to block missionpack pak0.pk3 models from the standard non-TA model list + // which doesn't handle their skin setting correctly + flags |= FLISTFLAG_IGNORE_TAPAK0; + } + */ + + { + filelist_query_t query = {extension, 0, flags}; + pFiles = list_files(path, &nFiles, &query); + } + + for (i = 0; i < nFiles; i++) + { + nLen = strlen(pFiles[i]) + 1; + if (nTotal + nLen + 1 < bufsize) + { + strcpy(listbuf, pFiles[i]); + listbuf += nLen; + nTotal += nLen; + } + else + { + nFiles = i; + break; + } + } + + FS_FreeFileList(pFiles); + + return nFiles; +} + +int FS_GetFilteredFiles(const char *path, const char *extension, const char *filter, char *listbuf, int bufsize) +{ + // path, extension, and filter may be null + int i; + int flags = 0; + int nFiles = 0; + int nTotal = 0; + int nLen; + char **pFiles = NULL; + FSC_ASSERT(listbuf); + + *listbuf = 0; + + { + filelist_query_t query = {extension, filter, flags}; + pFiles = list_files(path, &nFiles, &query); + } + + for (i = 0; i < nFiles; i++) + { + nLen = strlen(pFiles[i]) + 1; + if (nTotal + nLen + 1 < bufsize) + { + strcpy(listbuf, pFiles[i]); + listbuf += nLen; + nTotal += nLen; + } + else + { + nFiles = i; + break; + } + } + + FS_FreeFileList(pFiles); + + return nFiles; +} + +#endif // NEW_FILESYSTEM diff --git a/src/filesystem/fs_lookup.cpp b/src/filesystem/fs_lookup.cpp new file mode 100644 index 000000000..44ed4b778 --- /dev/null +++ b/src/filesystem/fs_lookup.cpp @@ -0,0 +1,1299 @@ +/* +=========================================================================== +Copyright (C) 1999-2005 Id Software, Inc. +Copyright (C) 2017 Noah Metzger (chomenor@gmail.com) + +This file is part of Quake III Arena source code. + +Quake III Arena source code is free software; you can redistribute it +and/or modify it under the terms of the GNU General Public License as +published by the Free Software Foundation; either version 2 of the License, +or (at your option) any later version. + +Quake III Arena source code is distributed in the hope that it will be +useful, but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with Quake III Arena source code; if not, write to the Free Software +Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA +=========================================================================== +*/ + +#ifdef NEW_FILESYSTEM +#include +#include "fslocal.h" + +#define ADD_STRING(string) fsc_stream_append_string(stream, string) +#define ADD_STRINGL(string) fsc_stream_append_string(&stream, string) + +typedef struct { + // File lookup + const char *qp_name; // null to disable file lookup + const char *qp_dir; + const char **qp_exts; + int extension_count; + + // Shader lookup + const char *shader_name; // null to disable shader lookup + + // Lookup flags + int lookup_flags; + + // Special + qboolean dll_query; + qboolean tremulous_core_game_query; +} lookup_query_t; + +#define RESFLAG_IN_DOWNLOAD_PK3 1 +#define RESFLAG_IN_CURRENT_MAP_PAK 2 +#define RESFLAG_FROM_DLL_QUERY 4 +#define RESFLAG_CASE_MISMATCH 8 +#define RESFLAG_AUXILIARY_SOURCEDIR 16 + +typedef struct { + // This should contain only static data, i.e. no pointers that expire when the query + // finishes, since it gets saved for debug queries + const fsc_file_t *file; + const fsc_shader_t *shader; + int core_pak_priority; + int server_pure_position; + int extension_position; + FS_ModType mod_type; + int flags; + + // Can be set to an error explanation to disable the resource during selection but still + // have it show up in the precedence debug listings. + const char *disabled; +} lookup_resource_t; + +typedef struct { + lookup_resource_t *resources; + int resource_count; + int resource_allocation; +} selection_output_t; + +typedef struct { + const fsc_file_t *file; + const fsc_shader_t *shader; +} query_result_t; + +/* ******************************************************************************** */ +// Resource construction - Converts file or shader to lookup resource +/* ******************************************************************************** */ + +static void configure_lookup_resource(const lookup_query_t *query, lookup_resource_t *resource) +{ + const char *resource_mod_dir = fsc_get_mod_dir(resource->file, &fs); + const fsc_file_direct_t *base_file = fsc_get_base_file(resource->file, &fs); + + // Determine mod dir match level + bool prioritize_fs_basegame = (query->lookup_flags & LOOKUPFLAG_PRIORITIZE_FS_BASEGAME) ? true : false; + resource->mod_type = fs_get_mod_type(resource_mod_dir, prioritize_fs_basegame); + + // Configure pk3-specific properties + if (resource->file->sourcetype == FSC_SOURCETYPE_PK3) + { + if (!(query->lookup_flags & LOOKUPFLAG_IGNORE_PURE_LIST)) + resource->server_pure_position = pk3_list_lookup(&connected_server_pure_list, base_file->pk3_hash); + if (base_file->f.flags & FSC_FILEFLAG_DLPK3) + resource->flags |= RESFLAG_IN_DOWNLOAD_PK3; + + if (resource->mod_type < MODTYPE_OVERRIDE_DIRECTORY) + { + // Sort core paks or the current map pak specially unless they are mixed into an active mod directory + resource->core_pak_priority = core_pk3_position(base_file->pk3_hash); + if (!(query->lookup_flags & LOOKUPFLAG_IGNORE_CURRENT_MAP) && base_file == current_map_pk3) + resource->flags |= RESFLAG_IN_CURRENT_MAP_PAK; + } + } + + // Special: Handle Tremulous core game dll query + if (query->tremulous_core_game_query) + { + // Core game dll must be located in trem13 mod directory + if (Q_stricmp(resource_mod_dir, BASEGAME_1_3)) + resource->disabled = "core game dll must be located in " BASEGAME_1_3 " directory"; + + // Apply appropriate core resource precedence value + resource->core_pak_priority = core_special_position("core_game_dll"); + } + + // Check for case-mismatched mod directories + for (const char *name : {BASEGAME_1_1, BASEGAME_GPP, BASEGAME_1_3, BASEGAME_OVERRIDE, + (const char *)fs_basegame->string, (const char *)current_mod_dir}) + { + if (!Q_stricmp(resource_mod_dir, name) && strcmp(resource_mod_dir, name)) + resource->flags |= RESFLAG_CASE_MISMATCH; + } + + // Check for auxiliary source directory + if (base_file && fs_sourcedirs[base_file->source_dir_id].auxiliary && !resource->server_pure_position) + { + resource->flags |= RESFLAG_AUXILIARY_SOURCEDIR; + } + + // Restrict source locations for settings (e.g. q3config.cfg, autoexec.cfg, or default.cfg) query + if (query->lookup_flags & LOOKUPFLAG_SETTINGS_FILE) + { + if (fs_mod_settings->integer && Q_stricmp(resource_mod_dir, (const char *)fs_basegame->string) && + resource->mod_type != MODTYPE_CURRENT_MOD) + { + resource->disabled = "settings config file can only be loaded from fs_basegame or current mod dir"; + } + if (!fs_mod_settings->integer && Q_stricmp(resource_mod_dir, (const char *)fs_basegame->string)) + { + resource->disabled = "settings config file can only be loaded from fs_basegame dir"; + } + } + + // Dll query handling + if (query->dll_query) + { + if (resource->file->sourcetype != FSC_SOURCETYPE_DIRECT) + { + resource->disabled = "dll files can only be loaded directly from disk"; + } + resource->flags |= RESFLAG_FROM_DLL_QUERY; + } + + // Disable files according to lookupflag sourcetype restrictions + if ((query->lookup_flags & LOOKUPFLAG_DIRECT_SOURCE_ONLY) && resource->file->sourcetype != FSC_SOURCETYPE_DIRECT) + { + resource->disabled = "blocking file due to direct_source_only flag"; + } + if ((query->lookup_flags & LOOKUPFLAG_PK3_SOURCE_ONLY) && resource->file->sourcetype != FSC_SOURCETYPE_PK3) + { + resource->disabled = "blocking file due to pk3_source_only flag"; + } + + // Disable files according to download folder restrictions + if ((query->lookup_flags & LOOKUPFLAG_NO_DOWNLOAD_FOLDER) && (resource->flags & RESFLAG_IN_DOWNLOAD_PK3)) + { + resource->disabled = "blocking file in download folder due to no_download_folder flag"; + } + + // Disable files blocked by fs_read_inactive_mods setting + if (fs_file_disabled(resource->file, FD_CHECK_READ_INACTIVE_MODS)) + { + resource->disabled = "blocking file from inactive mod dir due to fs_read_inactive_mods setting"; + } + + // Disable files not on pure list if connected to a pure server + if (!resource->server_pure_position && fs_connected_server_pure_state() == 1 && + !(query->lookup_flags & LOOKUPFLAG_IGNORE_PURE_LIST) && + !((query->lookup_flags & LOOKUPFLAG_PURE_ALLOW_DIRECT_SOURCE) && + resource->file->sourcetype == FSC_SOURCETYPE_DIRECT)) + { + resource->disabled = "connected to pure server and file is not on pure list"; + } +} + +static void file_to_lookup_resource(const lookup_query_t *query, const fsc_file_t *file, int extension_index, + qboolean case_mismatch, lookup_resource_t *resource) +{ + fsc_memset(resource, 0, sizeof(*resource)); + resource->file = file; + resource->shader = 0; + resource->extension_position = extension_index; + if (case_mismatch) + resource->flags |= RESFLAG_CASE_MISMATCH; + configure_lookup_resource(query, resource); +} + +static void shader_to_lookup_resource( + const lookup_query_t *query, const fsc_shader_t *shader, lookup_resource_t *resource) +{ + fsc_memset(resource, 0, sizeof(*resource)); + resource->file = (const fsc_file_t *)STACKPTR(shader->source_file_ptr); + resource->shader = shader; + configure_lookup_resource(query, resource); +} + +/* ******************************************************************************** */ +// Selection - Generates set of lookup resources for given query +/* ******************************************************************************** */ + +/* *** Selection Output Structure *** */ + +static void initialize_selection_output(selection_output_t *output) +{ + // Initialize output structure + fsc_memset(output, 0, sizeof(*output)); + output->resources = (lookup_resource_t *)Z_Malloc(20 * sizeof(*output->resources)); + output->resource_count = 0; + output->resource_allocation = 20; +} + +static void free_selection_output(selection_output_t *output) { Z_Free(output->resources); } + +static lookup_resource_t *allocate_lookup_resource(selection_output_t *output) +{ + if (output->resource_count >= output->resource_allocation) + { + // No more slots - double the allocation size + lookup_resource_t *new_resources = + (lookup_resource_t *)Z_Malloc(output->resource_count * 2 * sizeof(*output->resources)); + fsc_memcpy(new_resources, output->resources, output->resource_count * sizeof(*output->resources)); + Z_Free(output->resources); + output->resources = new_resources; + output->resource_allocation = output->resource_count * 2; + } + return &output->resources[output->resource_count++]; +} + +/* *** File Selection *** */ + +static qboolean string_match(const char *string1, const char *string2, qboolean *case_mismatch_out) +{ + if (!string1 || !string2) + return string1 == string2 ? qtrue : qfalse; + if (!Q_stricmp(string1, string2)) + { + if (!*case_mismatch_out && strcmp(string1, string2)) + *case_mismatch_out = qtrue; + return qtrue; + } + return qfalse; +} + +static int is_file_selected( + const fsc_file_t *file, const lookup_query_t *query, int *extension_index_out, qboolean *case_mismatch_out) +{ + // Returns 1 if selected, 0 otherwise + int i; + if (!fsc_is_file_enabled(file, &fs)) + return 0; + if (!string_match(query->qp_name, (const char *)STACKPTR(file->qp_name_ptr), case_mismatch_out)) + return 0; + if (!string_match(query->qp_dir, (const char *)STACKPTRN(file->qp_dir_ptr), case_mismatch_out)) + return 0; + + if (!query->extension_count) + return 1; + for (i = 0; i < query->extension_count; ++i) + { + if (string_match(query->qp_exts[i], (const char *)STACKPTRN(file->qp_ext_ptr), case_mismatch_out)) + { + *extension_index_out = i + 1; + return 1; + } + } + return 0; +} + +static void perform_file_selection(const lookup_query_t *query, selection_output_t *output) +{ + // Adds files matching criteria to output + fsc_hashtable_iterator_t hti; + fsc_stackptr_t file_ptr; + + fsc_hashtable_open(&fs.files, fsc_string_hash(query->qp_name, query->qp_dir), &hti); + while ((file_ptr = fsc_hashtable_next(&hti))) + { + const fsc_file_t *file = (const fsc_file_t *)STACKPTR(file_ptr); + int extension_index = 0; + qboolean case_mismatch = qfalse; + if (is_file_selected(file, query, &extension_index, &case_mismatch)) + { + lookup_resource_t *resource = allocate_lookup_resource(output); + file_to_lookup_resource(query, file, extension_index, case_mismatch, resource); + } + } +} + +/* *** Shader Selection *** */ + +static void perform_shader_selection(const lookup_query_t *query, selection_output_t *output) +{ + // Adds shaders matching criteria to output + fsc_hashtable_iterator_t hti; + fsc_stackptr_t shader_ptr; + + fsc_hashtable_open(&fs.shaders, fsc_string_hash(query->shader_name, 0), &hti); + while ((shader_ptr = fsc_hashtable_next(&hti))) + { + const fsc_shader_t *shader = (fsc_shader_t *)STACKPTR(shader_ptr); + if (!fsc_is_file_enabled((const fsc_file_t *)STACKPTR(shader->source_file_ptr), &fs)) + continue; + if (Q_stricmp((const char *)STACKPTR(shader->shader_name_ptr), query->shader_name)) + continue; + shader_to_lookup_resource(query, shader, allocate_lookup_resource(output)); + } +} + +/* *** Selection Main *** */ + +void perform_selection(const lookup_query_t *query, selection_output_t *output) +{ + // Caller is responsible for calling initialize_selection_output beforehand, + // and free_selection_output afterwards. + if (query->shader_name) + perform_shader_selection(query, output); + if (query->qp_name) + perform_file_selection(query, output); +} + +/* ******************************************************************************** */ +// Precedence - Used to select best lookup resource from set of lookup resources +/* ******************************************************************************** */ + +/* *** Support Functions *** */ + +static void resource_to_stream(const lookup_resource_t *resource, fsc_stream_t *stream) +{ + fs_file_to_stream(resource->file, stream, qtrue, qtrue, qtrue, resource->shader ? qfalse : qtrue); + + if (resource->shader) + { + ADD_STRING("->"); + ADD_STRING((const char *)STACKPTR(resource->shader->shader_name_ptr)); + } +} + +/* *** Precedence Rule Definitions *** */ + +#define PC_COMPARE(check_name) static int pc_cmp_##check_name(const lookup_resource_t *r1, const lookup_resource_t *r2) + +#define PC_DEBUG(check_name) \ + static void pc_dbg_##check_name( \ + const lookup_resource_t *high, const lookup_resource_t *low, int high_num, int low_num, fsc_stream_t *stream) + +// Return -1 if r1 has higher precedence, 1 if r2 has higher precedence, 0 if neither + +PC_COMPARE(resource_disabled) +{ + if (r1->disabled && !r2->disabled) + return 1; + if (r2->disabled && !r1->disabled) + return -1; + return 0; +} + +PC_DEBUG(resource_disabled) +{ + ADD_STRING(va("Resource %i was selected because resource %i is disabled: %s", high_num, low_num, low->disabled)); +} + +PC_COMPARE(auxiliary_sourcedir) +{ + if ((r1->flags & RESFLAG_AUXILIARY_SOURCEDIR) && !(r2->flags & RESFLAG_AUXILIARY_SOURCEDIR)) + return 1; + if ((r2->flags & RESFLAG_AUXILIARY_SOURCEDIR) && !(r1->flags & RESFLAG_AUXILIARY_SOURCEDIR)) + return -1; + return 0; +} + +PC_DEBUG(auxiliary_sourcedir) +{ + ADD_STRING( + va("Resource %i was selected because resource %i is in an auxiliary source directory.", high_num, low_num)); +} + +PC_COMPARE(special_shaders) +{ + qboolean r1_special = (r1->shader && (r1->mod_type >= MODTYPE_OVERRIDE_DIRECTORY || r1->core_pak_priority || + r1->server_pure_position)) + ? qtrue + : qfalse; + qboolean r2_special = (r2->shader && (r2->mod_type >= MODTYPE_OVERRIDE_DIRECTORY || r2->core_pak_priority || + r2->server_pure_position)) + ? qtrue + : qfalse; + + if (r1_special && !r2_special) + return -1; + if (r2_special && !r1_special) + return 1; + + return 0; +} + +PC_DEBUG(special_shaders) +{ + ADD_STRING(va( + "Resource %i was selected because it is classified as a special shader (from a core pak, the server pure list," + " the current mod dir, or the basemod dir) and resource %i is not.", + high_num, low_num)); +} + +PC_COMPARE(server_pure_position) +{ + if (r1->server_pure_position && !r2->server_pure_position) + return -1; + if (r2->server_pure_position && !r1->server_pure_position) + return 1; + + if (r1->server_pure_position < r2->server_pure_position) + return -1; + if (r2->server_pure_position < r1->server_pure_position) + return 1; + return 0; +} + +PC_DEBUG(server_pure_position) +{ + if (!low->server_pure_position) + { + ADD_STRING(va("Resource %i was selected because it is on the server pure list and resource %i is not.", + high_num, low_num)); + } + else + { + ADD_STRING( + va("Resource %i was selected because it has a lower server pure list position (%i) than resource %i (%i).", + high_num, high->server_pure_position, low_num, low->server_pure_position)); + } +} + +PC_COMPARE(basemod_or_current_mod_dir) +{ + if (r1->mod_type >= MODTYPE_OVERRIDE_DIRECTORY || r2->mod_type >= MODTYPE_OVERRIDE_DIRECTORY) + { + if (r1->mod_type > r2->mod_type) + return -1; + if (r2->mod_type > r1->mod_type) + return 1; + } + return 0; +} + +PC_DEBUG(basemod_or_current_mod_dir) +{ + ADD_STRING(va("Resource %i was selected because it is from ", high_num)); + if (high->mod_type == MODTYPE_CURRENT_MOD) + { + ADD_STRING(va("the current mod directory (%s)", fsc_get_mod_dir(high->file, &fs))); + } + else + { + ADD_STRING("the 'basemod' directory"); + } + ADD_STRING(va(" and resource %i is not. ", low_num)); +} + +PC_COMPARE(core_paks) +{ + if (r1->core_pak_priority > r2->core_pak_priority) + return -1; + if (r2->core_pak_priority > r1->core_pak_priority) + return 1; + return 0; +} + +PC_DEBUG(core_paks) +{ + ADD_STRING( + va("Resource %i was selected because it has a higher core pak rank than resource %i.", high_num, low_num)); +} + +PC_COMPARE(current_map_pak) +{ + if ((r1->flags & RESFLAG_IN_CURRENT_MAP_PAK) && !(r2->flags & RESFLAG_IN_CURRENT_MAP_PAK)) + return -1; + if ((r2->flags & RESFLAG_IN_CURRENT_MAP_PAK) && !(r1->flags & RESFLAG_IN_CURRENT_MAP_PAK)) + return 1; + return 0; +} + +PC_DEBUG(current_map_pak) +{ + ADD_STRING(va("Resource %i was selected because it is from the same pk3 as the current map and %i is not.", + high_num, low_num)); +} + +PC_COMPARE(mod_dir_priority) +{ + if (r1->mod_type > r2->mod_type) + return -1; + if (r2->mod_type > r1->mod_type) + return 1; + return 0; +} + +PC_DEBUG(mod_dir_priority) +{ + ADD_STRING(va("Resource %i was selected because it is from a higher priority mod directory", high_num)); +} + +PC_COMPARE(downloads_folder) +{ + if (!(r1->flags & RESFLAG_IN_DOWNLOAD_PK3) && (r2->flags & RESFLAG_IN_DOWNLOAD_PK3)) + return -1; + if (!(r2->flags & RESFLAG_IN_DOWNLOAD_PK3) && (r1->flags & RESFLAG_IN_DOWNLOAD_PK3)) + return 1; + return 0; +} + +PC_DEBUG(downloads_folder) +{ + ADD_STRING(va("Resource %i was selected because resource %i is in the downloads folder and resource %i is not.", + high_num, low_num, high_num)); +} + +PC_COMPARE(shader_over_image) +{ + if (r1->shader && !r2->shader) + return -1; + if (r2->shader && !r1->shader) + return 1; + return 0; +} + +PC_DEBUG(shader_over_image) +{ + ADD_STRING( + va("Resource %i was selected because it is a shader and resource %i is not a shader.", high_num, low_num)); +} + +PC_COMPARE(dll_over_qvm) +{ +#define QUALIFYING_DLL(resource) (resource->flags & RESFLAG_FROM_DLL_QUERY) + if (QUALIFYING_DLL(r1) && !QUALIFYING_DLL(r2)) + return -1; + if (QUALIFYING_DLL(r2) && !QUALIFYING_DLL(r1)) + return 1; + return 0; +} + +PC_DEBUG(dll_over_qvm) +{ + ADD_STRING(va("Resource %i was selected because it is a dll and resource %i is not a dll.", high_num, low_num)); +} + +PC_COMPARE(direct_over_pk3) +{ +#define PK3_LIKE_FILE(file) \ + (file->sourcetype == FSC_SOURCETYPE_PK3 || \ + (file->sourcetype == FSC_SOURCETYPE_DIRECT && ((fsc_file_direct_t *)file)->pk3dir_ptr)) + if (!PK3_LIKE_FILE(r1->file) && PK3_LIKE_FILE(r2->file)) + return -1; + if (!PK3_LIKE_FILE(r2->file) && PK3_LIKE_FILE(r1->file)) + return 1; + return 0; +} + +PC_DEBUG(direct_over_pk3) +{ + ADD_STRING( + va("Resource %i was selected because it is a file directly on the disk, while resource %i is inside a pk3.", + high_num, low_num)); +} + +PC_COMPARE(pk3_name_precedence) { return fs_compare_pk3_source(r1->file, r2->file); } + +PC_DEBUG(pk3_name_precedence) +{ + ADD_STRING( + va("Resource %i was selected because the pk3 containing it has lexicographically higher precedence " + "than the pk3 containing resource %i.", + high_num, low_num)); +} + +PC_COMPARE(extension_precedence) +{ + if (r1->extension_position < r2->extension_position) + return -1; + if (r2->extension_position < r1->extension_position) + return 1; + return 0; +} + +PC_DEBUG(extension_precedence) +{ + ADD_STRING( + va("Resource %i was selected because its extension has a higher precedence than " + "the extension of resource %i.", + high_num, low_num)); +} + +PC_COMPARE(source_dir_precedence) +{ + int r1_id = fs_get_source_dir_id(r1->file); + int r2_id = fs_get_source_dir_id(r2->file); + if (r1_id < r2_id) + return -1; + if (r2_id < r1_id) + return 1; + return 0; +} + +PC_DEBUG(source_dir_precedence) +{ + ADD_STRING(va( + "Resource %i was selected because it is from a higher precedence source directory (%s) than resource %i (%s)", + high_num, fs_get_source_dir_string(high->file), low_num, fs_get_source_dir_string(low->file))); +} + +PC_COMPARE(intra_pk3_position) +{ + if (r1->file->sourcetype != FSC_SOURCETYPE_PK3 || r2->file->sourcetype != FSC_SOURCETYPE_PK3) + return 0; + if (((fsc_file_frompk3_t *)r1->file)->header_position > ((fsc_file_frompk3_t *)r2->file)->header_position) + return -1; + if (((fsc_file_frompk3_t *)r2->file)->header_position > ((fsc_file_frompk3_t *)r1->file)->header_position) + return 1; + return 0; +} + +PC_DEBUG(intra_pk3_position) +{ + ADD_STRING(va("Resource %i was selected because it has a later position within the pk3 file than resource %i.", + high_num, low_num)); +} + +PC_COMPARE(intra_shaderfile_position) +{ + if (!r1->shader || !r2->shader) + return 0; + if (r1->shader->start_position < r2->shader->start_position) + return -1; + if (r2->shader->start_position < r1->shader->start_position) + return 1; + return 0; +} + +PC_DEBUG(intra_shaderfile_position) +{ + ADD_STRING( + va("Resource %i was selected because it has an earlier position within the shader file than resource %i.", + high_num, low_num)); +} + +PC_COMPARE(case_match) +{ + if (!(r1->flags & RESFLAG_CASE_MISMATCH) && (r2->flags & RESFLAG_CASE_MISMATCH)) + return -1; + if (!(r2->flags & RESFLAG_CASE_MISMATCH) && (r1->flags & RESFLAG_CASE_MISMATCH)) + return 1; + return 0; +} + +PC_DEBUG(case_match) +{ + ADD_STRING(va( + "Resource %i was selected because resource %i has a case discrepancy from the query and resource %i does not.", + high_num, low_num, high_num)); +} + +/* *** Precedence List & Sorting *** */ + +typedef struct { + const char *identifier; + int (*comparator)(const lookup_resource_t *r1, const lookup_resource_t *r2); + void (*debugfunction)( + const lookup_resource_t *high, const lookup_resource_t *low, int high_num, int low_num, fsc_stream_t *stream); +} precedence_check_t; + +#define ADD_CHECK(check) \ + { \ +#check, pc_cmp_##check, pc_dbg_##check \ + } +static const precedence_check_t precedence_checks[] = {ADD_CHECK(resource_disabled), ADD_CHECK(auxiliary_sourcedir), + ADD_CHECK(special_shaders), ADD_CHECK(server_pure_position), ADD_CHECK(basemod_or_current_mod_dir), + ADD_CHECK(core_paks), ADD_CHECK(current_map_pak), ADD_CHECK(mod_dir_priority), ADD_CHECK(downloads_folder), + ADD_CHECK(shader_over_image), ADD_CHECK(dll_over_qvm), ADD_CHECK(direct_over_pk3), ADD_CHECK(pk3_name_precedence), + ADD_CHECK(extension_precedence), ADD_CHECK(source_dir_precedence), ADD_CHECK(intra_pk3_position), + ADD_CHECK(intra_shaderfile_position), ADD_CHECK(case_match)}; + +static int precedence_comparator(const lookup_resource_t *resource1, const lookup_resource_t *resource2) +{ + int i; + int result; + for (i = 0; i < ARRAY_LEN(precedence_checks); ++i) + { + result = precedence_checks[i].comparator(resource1, resource2); + if (result) + return result; + } + // Use memory address as comparison of last resort + return resource1 < resource2 ? -1 : 1; +} + +static int precedence_comparator_qsort(const void *e1, const void *e2) +{ + return precedence_comparator((const lookup_resource_t *)e1, (const lookup_resource_t *)e2); +} + +static void selection_sort(selection_output_t *selection_output) +{ + qsort(selection_output->resources, selection_output->resource_count, sizeof(*selection_output->resources), + precedence_comparator_qsort); +} + +/* ******************************************************************************** */ +// Query processing - Runs selection/precedence oprations for a given lookup query +/* ******************************************************************************** */ + +/* *** Standard Lookup *** */ + +static void perform_lookup( + const lookup_query_t *queries, int query_count, qboolean protected_vm_lookup, query_result_t *output) +{ + int i; + selection_output_t selection_output; + lookup_resource_t *best_resource = 0; + + // Start with empty output + Com_Memset(output, 0, sizeof(*output)); + + // Perform selection + initialize_selection_output(&selection_output); + for (i = 0; i < query_count; ++i) + { + perform_selection(&queries[i], &selection_output); + } + + // If selection didn't receive anything, leave the output file null to indicate failure + if (!selection_output.resource_count) + { + free_selection_output(&selection_output); + return; + } + + if (protected_vm_lookup && fs_restrict_dlfolder->integer) + { + // Select the first resource that meets download folder restriction requirements + selection_sort(&selection_output); + for (i = 0; i < selection_output.resource_count; ++i) + { + if (selection_output.resources[i].flags & RESFLAG_IN_DOWNLOAD_PK3) + { + char buffer[FS_FILE_BUFFER_SIZE]; + qboolean trusted_hash = fs_check_trusted_vm_file(selection_output.resources[i].file); + if (!trusted_hash) + { + // Raise error if connected to pure server, otherwise just print a warning and skip + fs_file_to_buffer( + selection_output.resources[i].file, buffer, sizeof(buffer), qtrue, qtrue, qtrue, qfalse); + if (!com_sv_running->integer && fs_connected_server_pure_state()) + { + free_selection_output(&selection_output); + Com_Error(ERR_DROP, + "QVM file %s has an untrusted hash and was blocked due to your" + " fs_restrict_dlfolder setting. To remedy this, you may either:" + "\n- Move the pk3 out of the downloads folder. If you don't trust the server" + " you downloaded this file from, this may be a security risk." + "\n- Set fs_restrict_dlfolder to 0 to allow running all QVMs from" + " downloaded pk3s. This may lead to increased security risks.", + buffer); + } + Com_Printf( + "^3WARNING: QVM file %s has an untrusted hash and was blocked due to your" + " fs_restrict_dlfolder setting.\n", + buffer); + continue; + } + else if (fs_restrict_dlfolder->integer != 1) + { + // Raise error if connected to pure server, otherwise just print a warning and skip + fs_file_to_buffer( + selection_output.resources[i].file, buffer, sizeof(buffer), qtrue, qtrue, qtrue, qfalse); + if (!com_sv_running->integer && fs_connected_server_pure_state()) + { + free_selection_output(&selection_output); + Com_Error(ERR_DROP, + "QVM file %s has a trusted hash but was blocked due to your" + " fs_restrict_dlfolder setting. To remedy this, you may either:" + "\n- Move the pk3 out of the downloads folder. If you don't trust the server" + " you downloaded this file from, this may be a security risk." + "\n- Set fs_restrict_dlfolder to 1 to allow running trusted QVMs from" + " downloaded pk3s.", + buffer); + } + Com_Printf( + "^3WARNING: QVM file %s has a trusted hash but was blocked due to your" + " fs_restrict_dlfolder setting.\n", + buffer); + continue; + } + } + + // Have non-blocked file + best_resource = &selection_output.resources[i]; + break; + } + } + else + { + // Standard lookup; just pick the top resource + best_resource = &selection_output.resources[0]; + for (i = 1; i < selection_output.resource_count; ++i) + { + if (precedence_comparator(best_resource, &selection_output.resources[i]) > 0) + { + best_resource = &selection_output.resources[i]; + } + } + } + + if (best_resource && !best_resource->disabled) + { + output->file = best_resource->file; + output->shader = best_resource->shader; + } + + free_selection_output(&selection_output); +} + +/* *** Debug Query Storage *** */ + +static qboolean have_debug_selection = qfalse; +static selection_output_t debug_selection; + +/* *** Debug Lookup *** */ + +static void debug_lookup_flags_to_stream(int flags, fsc_stream_t *stream) +{ + const char *flag_strings[9] = {0}; + flag_strings[0] = (flags & LOOKUPFLAG_ENABLE_DDS) ? "enable_dds" : 0; + flag_strings[1] = (flags & LOOKUPFLAG_IGNORE_PURE_LIST) ? "ignore_pure_list" : 0; + flag_strings[2] = (flags & LOOKUPFLAG_PURE_ALLOW_DIRECT_SOURCE) ? "pure_allow_direct_source" : 0; + flag_strings[3] = (flags & LOOKUPFLAG_IGNORE_CURRENT_MAP) ? "ignore_current_map" : 0; + flag_strings[4] = (flags & LOOKUPFLAG_DIRECT_SOURCE_ONLY) ? "direct_source_only" : 0; + flag_strings[5] = (flags & LOOKUPFLAG_PK3_SOURCE_ONLY) ? "pk3_source_only" : 0; + flag_strings[6] = (flags & LOOKUPFLAG_SETTINGS_FILE) ? "settings_file" : 0; + flag_strings[7] = (flags & LOOKUPFLAG_NO_DOWNLOAD_FOLDER) ? "no_download_folder" : 0; + flag_strings[8] = (flags & LOOKUPFLAG_PRIORITIZE_FS_BASEGAME) ? "prioritize_fs_basegame" : 0; + fs_comma_separated_list(flag_strings, ARRAY_LEN(flag_strings), stream); +} + +static void debug_print_lookup_query(const lookup_query_t *query) +{ + char buffer[256]; + fsc_stream_t stream = {buffer, 0, sizeof(buffer), 0}; + Com_Printf(" path: %s%s%s\n", query->qp_dir ? query->qp_dir : "", query->qp_dir ? "/" : "", query->qp_name); + stream.position = 0; + fs_comma_separated_list(query->qp_exts, query->extension_count, &stream); + Com_Printf(" extensions: %s\n", stream.data); + Com_Printf(" shader: %s\n", query->shader_name ? query->shader_name : ""); + if (query->lookup_flags) + { + stream.position = 0; + debug_lookup_flags_to_stream(query->lookup_flags, &stream); + Com_Printf(" flags: %i (%s)\n", query->lookup_flags, stream.data); + } + else + { + Com_Printf(" flags: \n"); + } + Com_Printf(" dll_query: %s\n", query->dll_query ? "yes" : "no"); +} + +static void debug_lookup(const lookup_query_t *queries, int query_count, qboolean protected_vm_lookup) +{ + int i; + + // Print source queries + if (fs_debug_lookup->integer) + { + for (i = 0; i < query_count; ++i) + { + Com_Printf("Query %i\n", i + 1); + debug_print_lookup_query(&queries[i]); + Com_Printf("\n"); + } + } + + // Set global state + if (have_debug_selection) + free_selection_output(&debug_selection); + initialize_selection_output(&debug_selection); + + // Perform selection + for (i = 0; i < query_count; ++i) + { + perform_selection(&queries[i], &debug_selection); + } + selection_sort(&debug_selection); + have_debug_selection = qtrue; + + // Print element data + for (i = 0; i < debug_selection.resource_count; ++i) + { + char buffer[2048]; + fsc_stream_t stream = {buffer, 0, sizeof(buffer), 0}; + + ADD_STRINGL(va(" ^3Element %i: ^7", i + 1)); + resource_to_stream(&debug_selection.resources[i], &stream); + + if (protected_vm_lookup) + { + // Print extra hash data + unsigned char hash[32]; + calculate_file_sha256(debug_selection.resources[i].file, hash); + ADD_STRINGL("\nhash: "); + sha256_to_stream(hash, &stream); + + if (debug_selection.resources[i].file->qp_ext_ptr && + !Q_stricmp((const char *)STACKPTR(debug_selection.resources[i].file->qp_ext_ptr), "qvm")) + { + ADD_STRINGL(va("\ntrusted: %s", fs_check_trusted_vm_hash(hash) + ? "yes" + : "no; blocked in download folder if fs_restrict_dlfolder set")); + } + } + + Com_Printf("%s\n\n", buffer); + } + + if (!debug_selection.resource_count) + { + Com_Printf("No matching resources found.\n"); + } + else if (debug_selection.resources[0].disabled) + { + Com_Printf( + "No resource was selected because element 1 is disabled: %s\n", debug_selection.resources[0].disabled); + } +} + +/* *** Debug Comparison - Compares two resources, lists each test and result, and debug info for decisive test *** */ + +static void debug_resource_comparator(const lookup_resource_t *resource1, const lookup_resource_t *resource2, + int resource1_num, int resource2_num, fsc_stream_t *stream) +{ + int i; + int result; + int decisive_result = 0; + int decisive_test = -1; + int length; + + ADD_STRING("Check Result\n"); + ADD_STRING("------------------------------- ---------\n"); + + for (i = 0; i < ARRAY_LEN(precedence_checks); ++i) + { + // Write check name and spaces + ADD_STRING(precedence_checks[i].identifier); + length = strlen(precedence_checks[i].identifier); + while (length++ < 32) + ADD_STRING(" "); + + // Get result string + result = precedence_checks[i].comparator(resource1, resource2); + if (result && !decisive_result) + { + decisive_result = result; + decisive_test = i; + } + + // Write result identifier + if (result < 0) + ADD_STRING(va("resource %i", resource1_num)); + else if (result > 0) + ADD_STRING(va("resource %i", resource2_num)); + else + ADD_STRING("---"); + ADD_STRING("\n"); + } + + if (decisive_result) + { + const lookup_resource_t *high_resource, *low_resource; + int high_num, low_num; + if (decisive_result < 0) + { + high_resource = resource1; + low_resource = resource2; + high_num = resource1_num; + low_num = resource2_num; + } + else + { + high_resource = resource2; + low_resource = resource1; + high_num = resource2_num; + low_num = resource1_num; + } + + ADD_STRING("\n"); + precedence_checks[decisive_test].debugfunction(high_resource, low_resource, high_num, low_num, stream); + } +} + +void debug_resource_comparison(int resource1_position, int resource2_position) +{ + // Uses data from a previous lookup command (stored in debug_selection global variable) + // Input is the index (resource #) of two resources from that lookup + char data[65000]; + fsc_stream_t stream = {data, 0, sizeof(data), 0}; + + if (!have_debug_selection) + { + Com_Printf( + "This command must be preceded by a 'find_file', 'find_shader', 'find_sound', or 'find_vm' command.\n"); + return; + } + + if (resource1_position == resource2_position || resource1_position < 1 || resource2_position < 1 || + resource1_position > debug_selection.resource_count || resource2_position > debug_selection.resource_count) + { + Com_Printf("Resource numbers out of range.\n"); + return; + } + + debug_resource_comparator(&debug_selection.resources[resource1_position - 1], + &debug_selection.resources[resource2_position - 1], resource1_position, resource2_position, &stream); + + Com_Printf("%s\n", stream.data); +} + +/* ******************************************************************************** */ +// Wrapper functions - Generates query and calls query handling functions +/* ******************************************************************************** */ + +static void lookup_print_debug_file(const fsc_file_t *file) +{ + if (file) + { + char buffer[FS_FILE_BUFFER_SIZE]; + fs_file_to_buffer(file, buffer, sizeof(buffer), qtrue, qtrue, qtrue, qfalse); + FS_DPrintf("result: %s\n", buffer); + } + else + { + FS_DPrintf("result: \n"); + } +} + +static void lookup_print_debug_flags(int flags) +{ + if (flags) + { + char buffer[256]; + fsc_stream_t stream = {buffer, 0, sizeof(buffer), 0}; + debug_lookup_flags_to_stream(flags, &stream); + FS_DPrintf("flags: %i (%s)\n", flags, buffer); + } + else + { + FS_DPrintf("flags: \n"); + } +} + +const fsc_file_t *fs_general_lookup(const char *name, int lookup_flags, qboolean debug) +{ + lookup_query_t query; + char qpath_buffer[FSC_MAX_QPATH]; + const char *ext = 0; + query_result_t lookup_result; + FSC_ASSERT(name); + + Com_Memset(&query, 0, sizeof(query)); + query.lookup_flags = lookup_flags; + + // For compatibility purposes, support dropping one leading slash from qpath + // as per FS_FOpenFileReadDir in original filesystem + if (*name == '/' || *name == '\\') + ++name; + + fsc_process_qpath(name, qpath_buffer, &query.qp_dir, &query.qp_name, &ext); + query.qp_exts = &ext; + query.extension_count = ext ? 1 : 0; + + if (debug) + { + debug_lookup(&query, 1, qfalse); + return 0; + } + + perform_lookup(&query, 1, qfalse, &lookup_result); + if (fs_debug_lookup->integer) + { + FS_DPrintf("********** general lookup **********\n"); + fs_debug_indent_start(); + FS_DPrintf("name: %s\n", name); + lookup_print_debug_flags(lookup_flags); + lookup_print_debug_file(lookup_result.file); + fs_debug_indent_stop(); + } + return lookup_result.file; +} + +static void shader_or_image_lookup( + const char *name, qboolean image_only, int lookup_flags, query_result_t *output, qboolean debug) +{ + // Input name should be extension-free (call COM_StripExtension first) + lookup_query_t query; + char qpath_buffer[FSC_MAX_QPATH]; + const char *exts[] = {"dds", "png", "tga", "jpg", "jpeg", "pcx", "bmp"}; + + Com_Memset(&query, 0, sizeof(query)); + query.lookup_flags = lookup_flags; + if (!image_only) + query.shader_name = (char *)name; // const cast! + + // For compatibility purposes, support dropping one leading slash from qpath + // as per FS_FOpenFileReadDir in original filesystem + if (*name == '/' || *name == '\\') + ++name; + + fsc_process_qpath(name, qpath_buffer, &query.qp_dir, &query.qp_name, 0); + query.qp_exts = (lookup_flags & LOOKUPFLAG_ENABLE_DDS) ? exts : exts + 1; + query.extension_count = (lookup_flags & LOOKUPFLAG_ENABLE_DDS) ? ARRAY_LEN(exts) : ARRAY_LEN(exts) - 1; + + if (debug) + debug_lookup(&query, 1, qfalse); + else + perform_lookup(&query, 1, qfalse, output); +} + +const fsc_shader_t *fs_shader_lookup(const char *name, int lookup_flags, qboolean debug) +{ + // Input name should be extension-free (call COM_StripExtension first) + // Returns null if shader not found or image took precedence + query_result_t lookup_result; + FSC_ASSERT(name); + + shader_or_image_lookup(name, qfalse, lookup_flags, &lookup_result, debug); + if (debug) + return 0; + if (fs_debug_lookup->integer) + { + FS_DPrintf("********** shader lookup **********\n"); + fs_debug_indent_start(); + FS_DPrintf("name: %s\n", name); + lookup_print_debug_flags(lookup_flags); + lookup_print_debug_file(lookup_result.file); + fs_debug_indent_stop(); + } + return lookup_result.shader; +} + +const fsc_file_t *fs_image_lookup(const char *name, int lookup_flags, qboolean debug) +{ + // Input name should be extension-free (call COM_StripExtension first) + query_result_t lookup_result; + FSC_ASSERT(name); + + shader_or_image_lookup(name, qtrue, lookup_flags, &lookup_result, debug); + if (debug) + return 0; + if (fs_debug_lookup->integer) + { + FS_DPrintf("********** image lookup **********\n"); + fs_debug_indent_start(); + FS_DPrintf("name: %s\n", name); + lookup_print_debug_flags(lookup_flags); + lookup_print_debug_file(lookup_result.file); + fs_debug_indent_stop(); + } + return lookup_result.file; +} + +const fsc_file_t *fs_sound_lookup(const char *name, int lookup_flags, qboolean debug) +{ + // Input name should be extension-free (call COM_StripExtension first) + lookup_query_t query; + char qpath_buffer[FSC_MAX_QPATH]; + const char *exts[] = { + "wav", +#ifdef USE_CODEC_VORBIS + "ogg", +#endif +#ifdef USE_CODEC_OPUS + "opus", +#endif + }; + query_result_t lookup_result; + FSC_ASSERT(name); + + // For compatibility purposes, support dropping one leading slash from qpath + // as per FS_FOpenFileReadDir in original filesystem + if (*name == '/' || *name == '\\') + ++name; + + Com_Memset(&query, 0, sizeof(query)); + fsc_process_qpath(name, qpath_buffer, &query.qp_dir, &query.qp_name, 0); + query.qp_exts = exts; + query.extension_count = ARRAY_LEN(exts); + + if (debug) + { + debug_lookup(&query, 1, qfalse); + return 0; + } + + perform_lookup(&query, 1, qfalse, &lookup_result); + if (fs_debug_lookup->integer) + { + FS_DPrintf("********** sound lookup **********\n"); + fs_debug_indent_start(); + FS_DPrintf("name: %s\n", name); + lookup_print_debug_flags(lookup_flags); + lookup_print_debug_file(lookup_result.file); + fs_debug_indent_stop(); + } + return lookup_result.file; +} + +const fsc_file_t *fs_vm_lookup(const char *name, qboolean qvm_only, qboolean debug, qboolean *is_dll_out) +{ + // Returns a qvm or game dll file, or null if not found + // May throw ERR_DROP due to fs_restrict_dlfolder checks + const char *qvm_exts[] = {"qvm"}; + const char *dll_exts[] = {DLL_EXT + 1}; + lookup_query_t queries[3]; + char qpath_buffers[3][FSC_MAX_QPATH]; + int query_count = 1; + query_result_t lookup_result; + FSC_ASSERT(name); + + Com_Memset(queries, 0, sizeof(queries)); + queries[0].lookup_flags = LOOKUPFLAG_IGNORE_CURRENT_MAP; + fsc_process_qpath(va("vm/%s", name), qpath_buffers[0], &queries[0].qp_dir, &queries[0].qp_name, 0); + queries[0].qp_exts = qvm_exts; + queries[0].extension_count = ARRAY_LEN(qvm_exts); + + if (!qvm_only) + { + // Don't include ARCH_STRING in dll name as per GrangerHub standard + queries[1].lookup_flags = LOOKUPFLAG_IGNORE_CURRENT_MAP; + fsc_process_qpath(va("%s" /*ARCH_STRING*/, name), qpath_buffers[1], &queries[1].qp_dir, &queries[1].qp_name, 0); + queries[1].qp_exts = dll_exts; + queries[1].extension_count = ARRAY_LEN(dll_exts); + queries[1].dll_query = qtrue; + ++query_count; + + if (!Q_stricmp(name, "game")) + { + // Add core game dll for consideration + queries[2].lookup_flags = LOOKUPFLAG_IGNORE_CURRENT_MAP; + fsc_process_qpath(FS_CORE_GAME_DLL_NAME, qpath_buffers[2], &queries[2].qp_dir, &queries[2].qp_name, 0); + queries[2].qp_exts = dll_exts; + queries[2].extension_count = ARRAY_LEN(dll_exts); + queries[2].dll_query = qtrue; + queries[2].tremulous_core_game_query = qtrue; + ++query_count; + } + } + + if (debug) + { + debug_lookup(queries, query_count, qtrue); + return 0; + } + + perform_lookup(queries, query_count, qtrue, &lookup_result); + if (fs_debug_lookup->integer) + { + FS_DPrintf("********** dll/qvm lookup **********\n"); + fs_debug_indent_start(); + FS_DPrintf("name: %s\n", name); + FS_DPrintf("qvm only: %s\n", qvm_only ? "yes" : "no"); + lookup_print_debug_file(lookup_result.file); + fs_debug_indent_stop(); + } + + // Not elegant but should be adequate + if (is_dll_out) + *is_dll_out = lookup_result.file && lookup_result.file->qp_ext_ptr && + !Q_stricmp((const char *)STACKPTR(lookup_result.file->qp_ext_ptr), DLL_EXT + 1) + ? qtrue + : qfalse; + return lookup_result.file; +} + +#endif // NEW_FILESYSTEM diff --git a/src/filesystem/fs_main.cpp b/src/filesystem/fs_main.cpp new file mode 100644 index 000000000..dcd6d2f51 --- /dev/null +++ b/src/filesystem/fs_main.cpp @@ -0,0 +1,704 @@ +/* +=========================================================================== +Copyright (C) 1999-2005 Id Software, Inc. +Copyright (C) 2017 Noah Metzger (chomenor@gmail.com) + +This file is part of Quake III Arena source code. + +Quake III Arena source code is free software; you can redistribute it +and/or modify it under the terms of the GNU General Public License as +published by the Free Software Foundation; either version 2 of the License, +or (at your option) any later version. + +Quake III Arena source code is distributed in the hope that it will be +useful, but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with Quake III Arena source code; if not, write to the Free Software +Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA +=========================================================================== +*/ + +#ifdef NEW_FILESYSTEM +#include +#include "fslocal.h" + +/* ******************************************************************************** */ +// Filesystem State +/* ******************************************************************************** */ + +// The convention is that everything here can be accessed from anywhere in the +// filesystem code, but should only be modified through this file + +cvar_t *fs_dirs; +cvar_t *fs_mod_settings; +cvar_t *fs_index_cache; +cvar_t *fs_read_inactive_mods; +cvar_t *fs_list_inactive_mods; +cvar_t *fs_download_manifest; +cvar_t *fs_pure_manifest; +cvar_t *fs_redownload_across_mods; +cvar_t *fs_full_pure_validation; +cvar_t *fs_saveto_dlfolder; +cvar_t *fs_restrict_dlfolder; +cvar_t *fs_auto_refresh; + +cvar_t *fs_debug_state; +cvar_t *fs_debug_refresh; +cvar_t *fs_debug_fileio; +cvar_t *fs_debug_lookup; +cvar_t *fs_debug_references; +cvar_t *fs_debug_filelist; + +cvar_t *fs_basegame; +cvar_t *fs_force_profile; + +fs_source_directory_t fs_sourcedirs[FS_MAX_SOURCEDIRS]; +qboolean fs_read_only; // If false, fs_sourcedirs[0] is write directory + +fsc_filesystem_t fs; +static qboolean fs_initialized = qfalse; + +cvar_t *fs_game; +char current_mod_dir[FSC_MAX_MODDIR]; // Matched to fs_game when fs_set_mod_dir is called +const fsc_file_direct_t *current_map_pk3; +int checksum_feed; + +// Store connected server's sv_pure value here instead of relying on the cvar, +// because the cvar can be changed in console after connecting +int connected_server_sv_pure; +pk3_list_t connected_server_pure_list; + +// Current version profile used to prioritize file lookups +FS_Profile fs_connected_server_profile; + +/* ******************************************************************************** */ +// Filesystem State Accessors +/* ******************************************************************************** */ + +const char *FS_GetCurrentGameDir(void) +{ + // Returns mod dir, but with empty mod dir replaced by basegame + // Note: Potentially the usage in SV_RehashBans_f and SV_WriteBans could be changed + // to just use the non-SV file opening functions instead + if (*current_mod_dir) + return current_mod_dir; + return fs_basegame->string; +} + +const char *fs_pid_file_directory(void) +{ + if (fs_mod_settings->integer) + return FS_GetCurrentGameDir(); + return fs_basegame->string; +} + +qboolean FS_Initialized(void) +{ + // This might not be being used anywhere it could actually be false. Consider removing this + // function, or at least most of the calls to it, to reduce complexity. + return fs_initialized; +} + +int fs_connected_server_pure_state(void) +{ + // Returns 2 if semi-pure, 1 if pure, 0 if non-pure + if (!connected_server_pure_list.ht.element_count) + return 0; + if (connected_server_sv_pure == 2) + return 2; + return 1; +} + +FS_Profile fs_current_profile(void) +{ + if (*fs_force_profile->string) + { + if (!Q_stricmp(fs_force_profile->string, "13")) + return FS_PROFILE_1_3; + if (!Q_stricmp(fs_force_profile->string, "gpp")) + return FS_PROFILE_GPP; + if (!Q_stricmp(fs_force_profile->string, "11")) + return FS_PROFILE_1_1; + } + if (fs_connected_server_profile != FS_PROFILE_UNSET) + return fs_connected_server_profile; + return FS_PROFILE_1_3; +} + +/* ******************************************************************************** */ +// Filesystem State Modifiers +/* ******************************************************************************** */ + +void fs_register_current_map(const char *name) +{ + const fsc_file_t *bsp_file = fs_general_lookup(name, LOOKUPFLAG_IGNORE_CURRENT_MAP, qfalse); + if (!bsp_file || bsp_file->sourcetype != FSC_SOURCETYPE_PK3) + current_map_pk3 = 0; + else + current_map_pk3 = fsc_get_base_file(bsp_file, &fs); + + if (fs_debug_state->integer) + { + char buffer[FS_FILE_BUFFER_SIZE]; + if (current_map_pk3) + fs_file_to_buffer((fsc_file_t *)current_map_pk3, buffer, sizeof(buffer), qtrue, qtrue, qtrue, qfalse); + else + Q_strncpyz(buffer, "", sizeof(buffer)); + Com_Printf("fs_state: current_map_pk3 set to '%s'\n", buffer); + } +} + +void fs_set_connected_server_sv_pure_value(int sv_pure) +{ + connected_server_sv_pure = sv_pure; + if (fs_debug_state->integer) + { + Com_Printf("fs_state: connected_server_sv_pure set to %i\n", sv_pure); + } +} + +void fs_set_connected_server_profile(int alternateProtocol) +{ + if (alternateProtocol == 0) + fs_connected_server_profile = FS_PROFILE_1_3; + else if (alternateProtocol == 1) + fs_connected_server_profile = FS_PROFILE_GPP; + else + fs_connected_server_profile = FS_PROFILE_1_1; + if (fs_debug_state->integer) + { + Com_Printf("fs_state: connected_server_profile set to %s\n", fs_profile_to_string(fs_connected_server_profile)); + } +} + +void FS_PureServerSetLoadedPaks(const char *hash_list, const char *name_list) +{ + int i; + int count; + + pk3_list_free(&connected_server_pure_list); + pk3_list_initialize(&connected_server_pure_list, 100); + + Cmd_TokenizeString(hash_list); + count = Cmd_Argc(); + if (count > 4096) + count = 4096; // Sanity check + + for (i = 0; i < count; ++i) + { + pk3_list_insert(&connected_server_pure_list, atoi(Cmd_Argv(i))); + } + + if (fs_debug_state->integer) + Com_Printf("fs_state: connected_server_pure_list set to '%s'\n", hash_list); +} + +void fs_disconnect_cleanup(void) +{ + current_map_pk3 = 0; + connected_server_sv_pure = 0; + fs_connected_server_profile = FS_PROFILE_UNSET; + pk3_list_free(&connected_server_pure_list); + if (fs_debug_state->integer) + Com_Printf( + "fs_state: disconnect cleanup\n" + " > current_map_pk3 cleared\n" + " > connected_server_sv_pure set to 0\n" + " > connected_server_pure_list cleared\n" + " > connected_server_profile cleared\n"); +} + +static void generate_current_mod_dir(const char *source, char *target) +{ + // Converts mod dir to format used in current_mod_dir, with base directories and basemod + // replaced by empty string + // Target should be size FSC_MAX_MODDIR + fs_sanitize_mod_dir(source, target); + for (const char *name : {BASEGAME_1_1, BASEGAME_GPP, BASEGAME_1_3, BASEGAME_OVERRIDE}) + { + if (!Q_stricmp(target, name)) + *target = 0; + } +} + +static qboolean matches_current_mod_dir(const char *mod_dir) +{ + char generated_mod_dir[FSC_MAX_MODDIR]; + generate_current_mod_dir(mod_dir, generated_mod_dir); + return strcmp(current_mod_dir, generated_mod_dir) ? qfalse : qtrue; +} + +#ifndef STANDALONE +void Com_AppendCDKey(const char *filename); +void Com_ReadCDKey(const char *filename); +#endif + +void fs_set_mod_dir(const char *value, qboolean move_pid) +{ + char old_pid_dir[FSC_MAX_MODDIR]; + Q_strncpyz(old_pid_dir, fs_pid_file_directory(), sizeof(old_pid_dir)); + + // Set current_mod_dir + generate_current_mod_dir(value, current_mod_dir); + + /* + // Move pid file to new mod dir if necessary + if (move_pid && strcmp(old_pid_dir, fs_pid_file_directory())) + { + Sys_RemovePIDFile(old_pid_dir); + Sys_InitPIDFile(fs_pid_file_directory()); + } + */ + + // Read CD keys +#ifndef STANDALONE + if (!com_standalone->integer) + { + Com_ReadCDKey(BASEGAME); + if (strcmp(FS_GetCurrentGameDir(), BASEGAME)) + Com_AppendCDKey(FS_GetCurrentGameDir()); + } +#endif + + Cvar_Set("fs_game", current_mod_dir); + if (fs_debug_state->integer) + Com_Printf("fs_state: current_mod_dir set to %s\n", *current_mod_dir ? current_mod_dir : ""); +} + +qboolean FS_ConditionalRestart(int checksumFeed, qboolean disconnect) +{ + // This is used to activate a new mod dir, using a game restart if necessary to load + // new settings. It also sets the checksum feed (used for pure validation) and clears + // references from previous maps. + // Returns qtrue if restarting due to changed mod dir, qfalse otherwise + if (fs_debug_state->integer) + Com_Printf("fs_state: FS_ConditionalRestart invoked\n"); + FS_ClearPakReferences(0); + checksum_feed = checksumFeed; + + // Check for default.cfg here and attempt an ERR_DROP if it isn't found, to avoid getting + // an ERR_FATAL later due to broken pure list + if (!fs_general_lookup("default.cfg", LOOKUPFLAG_PURE_ALLOW_DIRECT_SOURCE, qfalse)) + { + Com_Error(ERR_DROP, "Failed to find default.cfg, assuming invalid configuration"); + } + + // Check if we need to do a restart to load new config files + if (fs_mod_settings->integer && !matches_current_mod_dir(fs_game->string)) + { + Com_GameRestart(checksumFeed, disconnect); + return qtrue; + } + + // Just update the mod dir + fs_set_mod_dir(fs_game->string, qtrue); + return qfalse; +} + +/* ******************************************************************************** */ +// Source Directory Determination +/* ******************************************************************************** */ + +static qboolean prepare_writable_directory(const char *directory) +{ + // Attempts to create directory and tests writability + // Returns qtrue if test passed, qfalse otherwise + char path[FS_MAX_PATH]; + void *fp; + + if (!fs_generate_path( + directory, "writetest.dat", 0, FS_CREATE_DIRECTORIES | FS_NO_SANITIZE, 0, 0, path, sizeof(path))) + return qfalse; + fp = fs_open_file(path, "wb"); + if (!fp) + return qfalse; + fsc_fclose(fp); + + fs_delete_file(path); + return qtrue; +} + +typedef struct { + fs_source_directory_t s; + int fs_dirs_position; // lower means higher priority + qboolean write_dir; +} temp_source_directory_t; + +static int compare_temp_source_dirs(const temp_source_directory_t *dir1, const temp_source_directory_t *dir2) +{ + // Put write dir first, and sort remaining directories according to position in fs_dirs cvar + if (dir1->s.active && !dir2->s.active) + return -1; + if (dir2->s.active && !dir1->s.active) + return 1; + if (dir1->write_dir && !dir2->write_dir) + return -1; + if (dir2->write_dir && !dir1->write_dir) + return 1; + if (dir1->fs_dirs_position < dir2->fs_dirs_position) + return -1; + return 1; +} + +static int compare_temp_source_dirs_qsort(const void *dir1, const void *dir2) +{ + return compare_temp_source_dirs((const temp_source_directory_t *)dir1, (const temp_source_directory_t *)dir2); +} + +void fs_initialize_sourcedirs(void) +{ + int i; + qboolean have_write_dir = qfalse; + char *fs_dirs_ptr; + temp_source_directory_t temp_dirs[FS_MAX_SOURCEDIRS]; + const char *homepath = Sys_DefaultHomePath(); + + // Initialize default path cvars + Cvar_Get("fs_homepath", homepath ? homepath : "", CVAR_INIT | CVAR_PROTECTED); + Cvar_Get("fs_basepath", Sys_DefaultInstallPath(), CVAR_INIT | CVAR_PROTECTED); +#ifdef __APPLE__ + Cvar_Get("fs_apppath", Sys_DefaultAppPath(), CVAR_INIT | CVAR_PROTECTED); +#endif + + // Generate temp_dirs based on fs_dirs entries + Com_Memset(temp_dirs, 0, sizeof(temp_dirs)); + fs_dirs_ptr = fs_dirs->string; + while (1) + { + qboolean write_flag = qfalse; + qboolean write_dir = qfalse; + qboolean auxiliary_dir = qfalse; + const char *token; + const char *path; + + // Get next token (source dir name) from fs_dirs + token = COM_ParseExt(&fs_dirs_ptr, qfalse); + if (!*token) + break; + + // Process prefixes + while (*token == '*' || *token == '#') + { + if (*token == '*') + write_flag = qtrue; + if (*token == '#') + auxiliary_dir = qtrue; + ++token; + } + if (!*token) + continue; + + // Determine path from cvar + path = Cvar_VariableString(token); + if (!*path) + continue; + + // Look for duplicate entry or next empty slot + for (i = 0; i < FS_MAX_SOURCEDIRS; ++i) + { + if (!temp_dirs[i].s.active) + break; + if (!Q_stricmp(token, temp_dirs[i].s.name)) + break; + } + if (i >= FS_MAX_SOURCEDIRS) + { + Com_Printf("WARNING: FS_MAX_SOURCEDIRS exceeded parsing fs_dirs\n"); + break; + } + if (temp_dirs[i].s.active) + { + Com_Printf("WARNING: Duplicate entry '%s' parsing fs_dirs\n", token); + continue; + } + + // If write flag is set and no write directory yet, test writability + if (write_flag && !have_write_dir) + { + Com_Printf("Checking if %s is writable...\n", token); + if (prepare_writable_directory(path)) + { + Com_Printf("Confirmed writable.\n"); + write_dir = qtrue; + have_write_dir = qtrue; + } + else + { + Com_Printf("Not writable due to failed write test.\n"); + } + } + + // Create entry in available slot + temp_dirs[i] = + (temp_source_directory_t){{CopyString(token), CopyString(path), qtrue, auxiliary_dir}, i, write_dir}; + } + + // Sort temp_dirs + qsort(temp_dirs, ARRAY_LEN(temp_dirs), sizeof(*temp_dirs), compare_temp_source_dirs_qsort); + + // Check for read-only mode + if (temp_dirs[0].write_dir) + { + fs_read_only = qfalse; + Com_Printf("Write directory: %s (%s)\n", temp_dirs[0].s.name, temp_dirs[0].s.path); + } + else + { + fs_read_only = qtrue; + Com_Printf("WARNING: No write directory selected. Filesystem in read-only mode.\n"); + } + + // Transfer entries from temp_dirs to fs_sourcedirs + for (i = 0; i < FS_MAX_SOURCEDIRS; ++i) + { + fs_sourcedirs[i] = temp_dirs[i].s; + if (fs_sourcedirs[i].active) + Com_Printf("Source directory %i%s: %s (%s)\n", i + 1, fs_sourcedirs[i].auxiliary ? " [AUX]" : "", + fs_sourcedirs[i].name, fs_sourcedirs[i].path); + } +} + +/* ******************************************************************************** */ +// Filesystem Refresh +/* ******************************************************************************** */ + +static void refresh_errorhandler(int id, const char *msg, void *current_element, void *context) +{ + if (fs_debug_refresh->integer) + { + const char *type = "general"; + if (id == FSC_ERROR_PK3FILE) + type = "pk3"; + if (id == FSC_ERROR_SHADERFILE) + type = "shader"; + Com_Printf("********** refresh %s error **********\n", type); + + if (current_element) + { + char buffer[FS_FILE_BUFFER_SIZE]; + fs_file_to_buffer((fsc_file_t *)current_element, buffer, sizeof(buffer), qtrue, qtrue, qtrue, qfalse); + Com_Printf("file: %s\n", buffer); + } + Com_Printf("message: %s\n", msg); + } +} + +#define NON_PK3_FILES(stats) (stats.total_file_count - stats.pk3_subfile_count - stats.valid_pk3_count) + +static void index_directory(const char *directory, int dir_id, qboolean quiet) +{ + fsc_errorhandler_t errorhandler = {refresh_errorhandler, 0}; + fsc_stats_t old_active_stats = fs.active_stats; + fsc_stats_t old_total_stats = fs.total_stats; + void *os_path = fsc_string_to_os_path(directory); + + fsc_load_directory(&fs, os_path, dir_id, &errorhandler); + fsc_free(os_path); + + if (!quiet) + { + Com_Printf("Indexed %i files in %i pk3s, %i other files, and %i shaders.\n", + fs.active_stats.pk3_subfile_count - old_active_stats.pk3_subfile_count, + fs.active_stats.valid_pk3_count - old_active_stats.valid_pk3_count, + NON_PK3_FILES(fs.active_stats) - NON_PK3_FILES(old_active_stats), + fs.active_stats.shader_count - old_active_stats.shader_count); + + Com_Printf("%i files in %i pk3s and %i shaders had not been previously indexed.\n", + fs.total_stats.pk3_subfile_count - old_total_stats.pk3_subfile_count, + fs.total_stats.valid_pk3_count - old_total_stats.valid_pk3_count, + fs.total_stats.shader_count - old_total_stats.shader_count); + } +} + +extern int com_frameNumber; +static int fs_refresh_frame = 0; + +void fs_refresh(qboolean quiet) +{ + int i; + if (fs_debug_refresh->integer) + quiet = qfalse; + if (!quiet) + Com_Printf("----- fs_refresh -----\n"); + + fsc_filesystem_reset(&fs); + + for (i = 0; i < FS_MAX_SOURCEDIRS; ++i) + { + if (!fs_sourcedirs[i].active) + continue; + if (!quiet) + Com_Printf("Indexing %s...\n", fs_sourcedirs[i].name); + index_directory(fs_sourcedirs[i].path, i, quiet); + } + + if (!quiet) + Com_Printf("Index memory usage at %iMB.\n", fsc_fs_size_estimate(&fs) / 1048576 + 1); + + fs_refresh_frame = com_frameNumber; + fs_readback_tracker_reset(); +} + +void fs_refresh_auto_ext(qboolean ignore_cvar, qboolean quiet) +{ + // Calls fs_refresh if enabled by settings, but maximum once per several frames + // to avoid redundant refreshes during load operations + // This can be called in any situation where there is a potential that files were + // added or changed on disk + // The ignore_cvar option is used for the fs_refresh console command, which should + // work for manual refreshes even if fs_auto_refresh setting is disabled + int frames_elapsed = com_frameNumber - fs_refresh_frame; + if (!ignore_cvar && !fs_auto_refresh->integer) + return; + if (frames_elapsed >= 0 && frames_elapsed < 5) + { + if (fs_debug_refresh->integer) + Com_Printf("Skipping redundant filesystem auto refresh.\n"); + return; + } + fs_refresh(quiet); +} + +void fs_refresh_auto(void) { fs_refresh_auto_ext(qfalse, qtrue); } + +/* ******************************************************************************** */ +// Filesystem Initialization +/* ******************************************************************************** */ + +static void *get_fscache_path(void) +{ + char path[FS_MAX_PATH]; + if (!fs_generate_path_sourcedir(0, "fscache.dat", 0, 0, 0, path, sizeof(path))) + return 0; + return fsc_string_to_os_path(path); +} + +void fs_indexcache_write(void) +{ + void *ospath = get_fscache_path(); + if (!ospath) + return; + fsc_cache_export_file(&fs, ospath, 0); + fsc_free(ospath); +} + +static qboolean fs_filesystem_refresh_tracked(void) +{ + // Calls fs_refresh, returns qtrue if enough changed to justify rewriting fscache.dat, + // returns qfalse otherwise + fsc_stats_t old_total_stats = fs.total_stats; + fs_refresh(qfalse); + if (fs.total_stats.valid_pk3_count - old_total_stats.valid_pk3_count > 20 || + fs.total_stats.pk3_subfile_count - old_total_stats.pk3_subfile_count > 5000 || + fs.total_stats.shader_file_count - old_total_stats.shader_file_count > 100 || + fs.total_stats.shader_count - old_total_stats.shader_count > 5000) + { + return qtrue; + } + return qfalse; +} + +static void fs_initialize_index(void) +{ + // Initialize the fs structure, using cache file if possible + qboolean cache_loaded = qfalse; + + if (fs_index_cache->integer) + { + void *path = get_fscache_path(); + Com_Printf("Loading fscache.dat...\n"); + if (path) + { + if (!fsc_cache_import_file(path, &fs, 0)) + cache_loaded = qtrue; + fsc_free(path); + } + + if (!cache_loaded) + Com_Printf("Failed to load fscache.dat.\n"); + } + + if (cache_loaded) + { + Com_Printf("Index data loaded for %i files, %i pk3s, and %i shaders.\n", + fs.files.utilization - fs.pk3_hash_lookup.utilization, fs.pk3_hash_lookup.utilization, + fs.shaders.utilization); + + if (fs_debug_refresh->integer) + { + Com_Printf( + "WARNING: Using index cache may prevent fs_debug_refresh error messages from being logged." + " For full debug info consider setting fs_index_cache to 0 or temporarily removing fscache.dat.\n"); + } + } + else + { + fsc_filesystem_initialize(&fs); + } +} + +static void fs_fatal_error_handler(const char *msg) { Com_Error(ERR_FATAL, "filesystem error: %s", msg); } + +void fs_startup(void) +{ + // Initial startup, should only be called once + Com_Printf("\n----- fs_startup -----\n"); + + fsc_register_fatal_error_handler(fs_fatal_error_handler); + +#ifdef __APPLE__ + fs_dirs = Cvar_Get("fs_dirs", "*fs_homepath fs_basepath fs_apppath", CVAR_INIT | CVAR_PROTECTED); +#else + fs_dirs = Cvar_Get("fs_dirs", "*fs_homepath fs_basepath", CVAR_INIT | CVAR_PROTECTED); +#endif + fs_mod_settings = Cvar_Get("fs_mod_settings", "0", CVAR_INIT); + fs_index_cache = Cvar_Get("fs_index_cache", "1", CVAR_INIT); + fs_read_inactive_mods = Cvar_Get("fs_read_inactive_mods", "1", CVAR_ARCHIVE); + fs_list_inactive_mods = Cvar_Get("fs_list_inactive_mods", "1", CVAR_ARCHIVE); + fs_download_manifest = + Cvar_Get("fs_download_manifest", "#mod_paks #cgame_pak #ui_pak #currentmap_pak #referenced_paks", CVAR_ARCHIVE); + fs_pure_manifest = Cvar_Get("fs_pure_manifest", "#mod_paks #base_paks #inactivemod_paks", CVAR_ARCHIVE); + fs_redownload_across_mods = Cvar_Get("fs_redownload_across_mods", "1", CVAR_ARCHIVE); + fs_full_pure_validation = Cvar_Get("fs_full_pure_validation", "0", CVAR_ARCHIVE); + fs_saveto_dlfolder = Cvar_Get("fs_saveto_dlfolder", "0", CVAR_ARCHIVE); + fs_restrict_dlfolder = Cvar_Get("fs_restrict_dlfolder", "0", CVAR_ARCHIVE); + fs_auto_refresh = Cvar_Get("fs_auto_refresh", "1", 0); + + fs_debug_state = Cvar_Get("fs_debug_state", "0", 0); + fs_debug_refresh = Cvar_Get("fs_debug_refresh", "0", 0); + fs_debug_fileio = Cvar_Get("fs_debug_fileio", "0", 0); + fs_debug_lookup = Cvar_Get("fs_debug_lookup", "0", 0); + fs_debug_references = Cvar_Get("fs_debug_references", "0", 0); + fs_debug_filelist = Cvar_Get("fs_debug_filelist", "0", 0); + + fs_basegame = Cvar_Get("fs_basegame", FS_BASEGAME_DEFAULT, CVAR_INIT); + fs_force_profile = Cvar_Get("fs_force_profile", "", 0); + + Cvar_Get("new_filesystem", "1", CVAR_ROM); // Enables new filesystem calls in renderer + + fs_initialize_sourcedirs(); + fs_initialize_index(); + + fs_game = Cvar_Get("fs_game", "", CVAR_INIT | CVAR_SYSTEMINFO); + fs_set_mod_dir(fs_game->string, qfalse); + + Com_Printf("\n"); + if (fs_filesystem_refresh_tracked() && fs_index_cache->integer && !fs_read_only) + { + Com_Printf("Writing fscache.dat due to updated files...\n"); + fs_indexcache_write(); + } + Com_Printf("\n"); + + fs_register_commands(); + fs_initialized = qtrue; + +#ifndef STANDALONE + fs_check_core_paks(); +#endif +} + +#endif // NEW_FILESYSTEM diff --git a/src/filesystem/fs_misc.cpp b/src/filesystem/fs_misc.cpp new file mode 100644 index 000000000..82b5ed838 --- /dev/null +++ b/src/filesystem/fs_misc.cpp @@ -0,0 +1,1083 @@ +/* +=========================================================================== +Copyright (C) 1999-2005 Id Software, Inc. +Copyright (C) 2017 Noah Metzger (chomenor@gmail.com) + +This file is part of Quake III Arena source code. + +Quake III Arena source code is free software; you can redistribute it +and/or modify it under the terms of the GNU General Public License as +published by the Free Software Foundation; either version 2 of the License, +or (at your option) any later version. + +Quake III Arena source code is distributed in the hope that it will be +useful, but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with Quake III Arena source code; if not, write to the Free Software +Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA +=========================================================================== +*/ + +#ifdef NEW_FILESYSTEM +#include "fslocal.h" + +/* ******************************************************************************** */ +// Indented Debug Print Support +/* ******************************************************************************** */ + +// This section is used to support indented prints for the cvar-enabled debug logging options +// to make the output more readable, especially if there are nested calls to functions that +// produce cluster-type prints +// Theoretically the level could be messed up by Com_Error, but since it's a pretty obscure +// scenario and this is ONLY used for cvar-enabled debug prints I'm not bothering with it + +static int fs_debug_indent_level = 0; + +void fs_debug_indent_start(void) { ++fs_debug_indent_level; } + +void fs_debug_indent_stop(void) +{ + --fs_debug_indent_level; + if (fs_debug_indent_level < 0) + { + Com_Printf("WARNING: Negative filesystem debug increment\n"); + fs_debug_indent_level = 0; + } +} + +void QDECL FS_DPrintf(const char *fmt, ...) +{ + va_list argptr; + char msg[MAXPRINTMSG]; + unsigned int indent = (unsigned int)fs_debug_indent_level; + char spaces[16] = " "; + + va_start(argptr, fmt); + Q_vsnprintf(msg, sizeof(msg), fmt, argptr); + va_end(argptr); + + if (indent > 4) + indent = 4; + spaces[indent * 2] = 0; + Com_Printf("%s%s", spaces, msg); +} + +/* ******************************************************************************** */ +// Hash Table +/* ******************************************************************************** */ + +void fs_hashtable_initialize(fs_hashtable_t *hashtable, int bucket_count) +{ + // Valid for an uninitialized hash table + FSC_ASSERT(hashtable); + FSC_ASSERT(bucket_count > 0); + hashtable->bucket_count = bucket_count; + hashtable->buckets = (fs_hashtable_entry_t **)Z_Malloc(sizeof(fs_hashtable_entry_t *) * bucket_count); + hashtable->element_count = 0; +} + +void fs_hashtable_insert(fs_hashtable_t *hashtable, fs_hashtable_entry_t *entry, unsigned int hash) +{ + // Valid for an initialized hash table + int index; + FSC_ASSERT(hashtable); + FSC_ASSERT(hashtable->bucket_count > 0); + FSC_ASSERT(entry); + index = hash % hashtable->bucket_count; + entry->next = hashtable->buckets[index]; + hashtable->buckets[index] = entry; + ++hashtable->element_count; +} + +fs_hashtable_iterator_t fs_hashtable_iterate(fs_hashtable_t *hashtable, unsigned int hash, qboolean iterate_all) +{ + // Valid for an initialized or uninitialized (zeroed) hashtable + fs_hashtable_iterator_t iterator; + FSC_ASSERT(hashtable); + iterator.ht = hashtable; + if (!hashtable->bucket_count || iterate_all) + { + iterator.current_bucket = 0; + iterator.bucket_limit = hashtable->bucket_count; + } + else + { + iterator.current_bucket = hash % hashtable->bucket_count; + iterator.bucket_limit = iterator.current_bucket + 1; + } + iterator.current_entry = 0; + return iterator; +} + +void *fs_hashtable_next(fs_hashtable_iterator_t *iterator) +{ + fs_hashtable_entry_t *entry = iterator->current_entry; + while (!entry) + { + if (iterator->current_bucket >= iterator->bucket_limit) + return 0; + entry = iterator->ht->buckets[iterator->current_bucket++]; + } + iterator->current_entry = entry->next; + return entry; +} + +static void fs_hashtable_free_entries(fs_hashtable_t *hashtable, void (*free_entry)(fs_hashtable_entry_t *entry)) +{ + // Valid for an initialized or uninitialized (zeroed) hashtable + fs_hashtable_iterator_t it = fs_hashtable_iterate(hashtable, 0, qtrue); + fs_hashtable_entry_t *entry; + if (free_entry) + while ((entry = (fs_hashtable_entry_t *)fs_hashtable_next(&it))) + free_entry(entry); + else + while ((entry = (fs_hashtable_entry_t *)fs_hashtable_next(&it))) + Z_Free(entry); +} + +void fs_hashtable_free(fs_hashtable_t *hashtable, void (*free_entry)(fs_hashtable_entry_t *entry)) +{ + // Valid for an initialized or uninitialized (zeroed) hashtable + FSC_ASSERT(hashtable); + fs_hashtable_free_entries(hashtable, free_entry); + if (hashtable->buckets) + Z_Free(hashtable->buckets); + Com_Memset(hashtable, 0, sizeof(*hashtable)); +} + +void fs_hashtable_reset(fs_hashtable_t *hashtable, void (*free_entry)(fs_hashtable_entry_t *entry)) +{ + // Valid for an initialized hash table + FSC_ASSERT(hashtable); + fs_hashtable_free_entries(hashtable, free_entry); + Com_Memset(hashtable->buckets, 0, sizeof(*hashtable->buckets) * hashtable->bucket_count); + hashtable->element_count = 0; +} + +/* ******************************************************************************** */ +// Pk3 List +/* ******************************************************************************** */ + +// The pk3 list structure maps pk3 hashes to index value +// First pk3 inserted has index 1, second pk3 has index 2, etc. +// If same hash is inserted multiple times, the first index will be used + +void pk3_list_initialize(pk3_list_t *pk3_list, unsigned int bucket_count) +{ + FSC_ASSERT(pk3_list); + fs_hashtable_initialize(&pk3_list->ht, bucket_count); +} + +int pk3_list_lookup(const pk3_list_t *pk3_list, unsigned int hash) +{ + fs_hashtable_iterator_t it; + pk3_list_entry_t *entry; + FSC_ASSERT(pk3_list); + it = fs_hashtable_iterate((fs_hashtable_t *)&pk3_list->ht, hash, qfalse); + while ((entry = (pk3_list_entry_t *)fs_hashtable_next(&it))) + { + if (entry->hash == hash) + { + return entry->position; + } + } + return 0; +} + +void pk3_list_insert(pk3_list_t *pk3_list, unsigned int hash) +{ + pk3_list_entry_t *new_entry; + FSC_ASSERT(pk3_list); + if (pk3_list_lookup(pk3_list, hash)) + return; + new_entry = (pk3_list_entry_t *)S_Malloc(sizeof(pk3_list_entry_t)); + fs_hashtable_insert(&pk3_list->ht, &new_entry->hte, hash); + new_entry->hash = hash; + new_entry->position = pk3_list->ht.element_count; +} + +void pk3_list_free(pk3_list_t *pk3_list) +{ + FSC_ASSERT(pk3_list); + fs_hashtable_free(&pk3_list->ht, 0); +} + +/* ******************************************************************************** */ +// File helper functions +/* ******************************************************************************** */ + +const char *fs_file_extension(const fsc_file_t *file) +{ + // Returns null for no extension + FSC_ASSERT(file); + return (const char *)STACKPTRN(file->qp_ext_ptr); +} + +qboolean fs_files_from_same_pk3(const fsc_file_t *file1, const fsc_file_t *file2) +{ + // Returns qtrue if both files are located in the same pk3, qfalse otherwise + // Used by renderer for md3 lod handling + if (!file1 || !file2 || file1->sourcetype != FSC_SOURCETYPE_PK3 || file2->sourcetype != FSC_SOURCETYPE_PK3 || + ((fsc_file_frompk3_t *)file1)->source_pk3 != ((fsc_file_frompk3_t *)file2)->source_pk3) + return qfalse; + return qtrue; +} + +int fs_get_source_dir_id(const fsc_file_t *file) +{ + const fsc_file_direct_t *base_file; + FSC_ASSERT(file); + base_file = fsc_get_base_file(file, &fs); + if (base_file) + return base_file->source_dir_id; + return -1; +} + +const char *fs_get_source_dir_string(const fsc_file_t *file) +{ + int id = fs_get_source_dir_id(file); + if (id >= 0 && id < FS_MAX_SOURCEDIRS && fs_sourcedirs[id].active) + return fs_sourcedirs[id].name; + return "unknown"; +} + +void fs_file_to_stream(const fsc_file_t *file, fsc_stream_t *stream, qboolean include_source_dir, qboolean include_mod, + qboolean include_pk3_origin, qboolean include_size) +{ + FSC_ASSERT(file && stream); + if (include_source_dir) + { + fsc_stream_append_string(stream, fs_get_source_dir_string(file)); + fsc_stream_append_string(stream, "->"); + } + fsc_file_to_stream(file, stream, &fs, include_mod, include_pk3_origin); + + if (include_size) + { + char buffer[24]; + Com_sprintf(buffer, sizeof(buffer), " (%i bytes)", file->filesize); + fsc_stream_append_string(stream, buffer); + } +} + +void fs_file_to_buffer(const fsc_file_t *file, char *buffer, unsigned int buffer_size, qboolean include_source_dir, + qboolean include_mod, qboolean include_pk3_origin, qboolean include_size) +{ + fsc_stream_t stream = {buffer, 0, buffer_size}; + FSC_ASSERT(file && buffer); + fs_file_to_stream(file, &stream, include_source_dir, include_mod, include_pk3_origin, include_size); +} + +void fs_print_file_location(const fsc_file_t *file) +{ + char name_buffer[FS_FILE_BUFFER_SIZE]; + char source_buffer[FS_FILE_BUFFER_SIZE]; + FSC_ASSERT(file); + fs_file_to_buffer(file, name_buffer, sizeof(name_buffer), qfalse, qfalse, qfalse, qfalse); + if (file->sourcetype == FSC_SOURCETYPE_PK3) + { + fs_file_to_buffer((const fsc_file_t *)fsc_get_base_file(file, &fs), source_buffer, sizeof(source_buffer), qtrue, + qtrue, qfalse, qfalse); + Com_Printf("File %s found in %s\n", name_buffer, source_buffer); + } + else if (file->sourcetype == FSC_SOURCETYPE_DIRECT) + { + fs_file_to_buffer(file, source_buffer, sizeof(source_buffer), qtrue, qtrue, qfalse, qfalse); + Com_Printf("File %s found at %s\n", name_buffer, source_buffer); + } + else + Com_Printf("File %s has unknown sourcetype\n", name_buffer); +} + +/* ******************************************************************************** */ +// File disabled check +/* ******************************************************************************** */ + +static int get_pk3_list_position(const fsc_file_t *file) +{ + if (file->sourcetype != FSC_SOURCETYPE_PK3) + return 0; + return pk3_list_lookup(&connected_server_pure_list, fsc_get_base_file(file, &fs)->pk3_hash); +} + +static qboolean inactive_mod_file_disabled(const fsc_file_t *file, int level) +{ + // Check if a file is disabled by inactive mod settings + + // Allow file if full inactive mod searching is enabled + if (level >= 2) + return qfalse; + + // Allow file if not in inactive mod directory + if (fs_get_mod_type(fsc_get_mod_dir(file, &fs)) > MODTYPE_INACTIVE) + return qfalse; + + // For setting 1, also allow files from core paks or on pure list + if (level == 1) + { + const fsc_file_direct_t *base_file = fsc_get_base_file(file, &fs); + if (base_file) + { + if (pk3_list_lookup(&connected_server_pure_list, base_file->pk3_hash)) + return qfalse; + if (core_pk3_position(base_file->pk3_hash)) + return qfalse; + } + } + + return qtrue; +} + +int fs_file_disabled(const fsc_file_t *file, int checks) +{ + // This function is used to perform various checks for whether a file should be used by the filesystem + // Returns value of one of the triggering checks if file is disabled, null otherwise + FSC_ASSERT(file); + + // File disabled check - blocks files disabled in the file index (they should never be accessed) + if ((checks & FD_CHECK_FILE_ENABLED) && !fsc_is_file_enabled(file, &fs)) + return FD_CHECK_FILE_ENABLED; + + // Pure list check - blocks files disabled by pure settings of server we are connected to + if ((checks & FD_CHECK_PURE_LIST) && fs_connected_server_pure_state() == 1) + { + if (!get_pk3_list_position(file)) + return FD_CHECK_PURE_LIST; + } + + // Search inactive mods check - blocks files disabled by inactive mod settings for file reading + if ((checks & FD_CHECK_READ_INACTIVE_MODS) && inactive_mod_file_disabled(file, fs_read_inactive_mods->integer)) + { + return FD_CHECK_READ_INACTIVE_MODS; + } + + // List inactive mods check - blocks files disabled by inactive mod settings for file listing + if (checks & FD_CHECK_LIST_INACTIVE_MODS) + { + // Use read_inactive_mods setting if it is lower, because it doesn't make sense to list unreadable files + int list_inactive_mods_level = fs_read_inactive_mods->integer < fs_list_inactive_mods->integer + ? fs_read_inactive_mods->integer + : fs_list_inactive_mods->integer; + if (inactive_mod_file_disabled(file, list_inactive_mods_level)) + return FD_CHECK_LIST_INACTIVE_MODS; + } + + // List auxiliary sourcedir check - blocks files from auxiliary source directories for file listing + if (checks & FD_CHECK_LIST_AUXILIARY_SOURCEDIR) + { + const fsc_file_direct_t *base_file = fsc_get_base_file(file, &fs); + if (base_file && fs_sourcedirs[base_file->source_dir_id].auxiliary && !get_pk3_list_position(file)) + { + return FD_CHECK_LIST_AUXILIARY_SOURCEDIR; + } + } + + return 0; +} + +/* ******************************************************************************** */ +// File Sorting Functions +/* ******************************************************************************** */ + +// The lookup, file list, and reference modules have their own sorting systems due +// to differences in requirements. Some sorting logic and functions that are shared +// between multiple modules are included here. + +static const unsigned char *get_string_sort_table(void) +{ + // The table maps characters to a precedence value + // higher value = higher precedence + qboolean initialized = qfalse; + static unsigned char table[256]; + + if (!initialized) + { + int i; + unsigned char value = 250; + for (i = 'z'; i >= 'a'; --i) + table[i] = value--; + value = 250; + for (i = 'Z'; i >= 'A'; --i) + table[i] = value--; + for (i = '9'; i >= '0'; --i) + table[i] = value--; + for (i = 255; i >= 0; --i) + if (!table[i]) + table[i] = value--; + initialized = qtrue; + } + + return table; +} + +static unsigned int server_pure_precedence(const fsc_file_t *file) +{ + if (file->sourcetype == FSC_SOURCETYPE_PK3) + { + // Pure list stores pk3s by position, with index 1 at highest priority, + // so index values need to be inverted to get precedence + unsigned int index = pk3_list_lookup(&connected_server_pure_list, fsc_get_base_file(file, &fs)->pk3_hash); + if (index) + return ~index; + } + return 0; +} + +static unsigned int mod_dir_precedence(FS_ModType mod_type) +{ + if (mod_type >= MODTYPE_OVERRIDE_DIRECTORY) + return (unsigned int)mod_type; + return 0; +} + +static unsigned int core_pak_precedence(const fsc_file_t *file, FS_ModType mod_type) +{ + if (mod_type < MODTYPE_OVERRIDE_DIRECTORY) + { + const fsc_file_direct_t *base_file = fsc_get_base_file(file, &fs); + if (base_file) + return core_pk3_position(base_file->pk3_hash); + } + return 0; +} + +static unsigned int basegame_dir_precedence(FS_ModType mod_type) { return (unsigned int)mod_type; } + +void fs_write_sort_string(const char *string, fsc_stream_t *output, qboolean prioritize_shorter) +{ + // Set prioritize_shorter true to prioritize shorter strings (i.e. "abc" over "abcd") + const unsigned char *sort_table = get_string_sort_table(); + while (*string && output->position < output->size) + { + output->data[output->position++] = (char)sort_table[*(unsigned char *)(string++)]; + } + if (output->position < output->size) + output->data[output->position++] = prioritize_shorter ? (char)(unsigned char)255 : 0; +} + +void fs_write_sort_filename(const fsc_file_t *file, fsc_stream_t *output) +{ + // Write sort key of the file itself + char buffer[FS_FILE_BUFFER_SIZE]; + fs_file_to_buffer(file, buffer, sizeof(buffer), qfalse, qfalse, qfalse, qfalse); + fs_write_sort_string(buffer, output, qfalse); +} + +static void write_sort_pk3_source_filename(const fsc_file_t *file, fsc_stream_t *output) +{ + // Write sort key of the pk3 file or pk3dir the file came from + if (file->sourcetype == FSC_SOURCETYPE_DIRECT && ((fsc_file_direct_t *)file)->pk3dir_ptr) + { + fs_write_sort_string((const char *)STACKPTR(((fsc_file_direct_t *)file)->pk3dir_ptr), output, qfalse); + fs_write_sort_value(1, output); + } + else if (file->sourcetype == FSC_SOURCETYPE_PK3) + { + const fsc_file_direct_t *source_pk3 = fsc_get_base_file(file, &fs); + fs_write_sort_string((const char *)STACKPTR(source_pk3->f.qp_name_ptr), output, qfalse); + fs_write_sort_value(0, output); + } +} + +void fs_write_sort_value(unsigned int value, fsc_stream_t *output) +{ + static volatile int test = 1; + if (*(char *)&test) + { + value = ((value << 8) & 0xFF00FF00) | ((value >> 8) & 0xFF00FF); + value = (value << 16) | (value >> 16); + } + if (output->position + 3 < output->size) + { + *((unsigned int *)(output->data + output->position)) = value; + output->position += 4; + } +} + +void fs_generate_core_sort_key(const fsc_file_t *file, fsc_stream_t *output, qboolean use_server_pure_list) +{ + // This is a rough version of the lookup precedence for reference and file listing purposes + // This sorts the mod/pk3 origin of the file, but not the actual file name, or the source directory + // since the file list system handles file names separately and currently ignores source directory + FS_ModType mod_type = fs_get_mod_type(fsc_get_mod_dir(file, &fs)); + if (use_server_pure_list) + fs_write_sort_value(server_pure_precedence(file), output); + fs_write_sort_value(mod_dir_precedence(mod_type), output); + fs_write_sort_value(core_pak_precedence(file, mod_type), output); + fs_write_sort_value(basegame_dir_precedence(mod_type), output); + // Deprioritize download folder pk3s, whether the flag is set for this file or this file's source pk3 + fs_write_sort_value( + (file->flags & FSC_FILEFLAG_DLPK3) || + (file->sourcetype == FSC_SOURCETYPE_PK3 && (fsc_get_base_file(file, &fs)->f.flags & FSC_FILEFLAG_DLPK3)) + ? 0 + : 1, + output); + if (file->sourcetype == FSC_SOURCETYPE_PK3 || + (file->sourcetype == FSC_SOURCETYPE_DIRECT && ((fsc_file_direct_t *)file)->pk3dir_ptr)) + { + fs_write_sort_value(0, output); + write_sort_pk3_source_filename(file, output); + fs_write_sort_value( + (file->sourcetype == FSC_SOURCETYPE_PK3) ? ~((fsc_file_frompk3_t *)file)->header_position : ~0u, output); + } + else + { + fs_write_sort_value(1, output); + } +} + +int fs_compare_pk3_source(const fsc_file_t *file1, const fsc_file_t *file2) +{ + char buffer1[1024]; + char buffer2[1024]; + fsc_stream_t stream1 = {buffer1, 0, sizeof(buffer1), qfalse}; + fsc_stream_t stream2 = {buffer2, 0, sizeof(buffer2), qfalse}; + write_sort_pk3_source_filename(file1, &stream1); + write_sort_pk3_source_filename(file2, &stream2); + return fsc_memcmp( + stream2.data, stream1.data, stream1.position < stream2.position ? stream1.position : stream2.position); +} + +/* ******************************************************************************** */ +// Misc Functions +/* ******************************************************************************** */ + +void fs_execute_config_file(const char *name, fs_config_type_t config_type, cbufExec_t exec_type, qboolean quiet) +{ + char *data; + unsigned int size = 0; + + if (com_journalDataFile && com_journal->integer == 2) + { + // In journal playback mode, try to load config files from journal data file + Com_Printf("execing %s from journal data file\n", name); + data = fs_read_journal_data(); + if (!data) + { + Com_Printf("couldn't exec %s - not present in journal\n", name); + return; + } + } + else + { + const fsc_file_t *file; + int lookup_flags = + LOOKUPFLAG_PURE_ALLOW_DIRECT_SOURCE | LOOKUPFLAG_IGNORE_CURRENT_MAP | LOOKUPFLAG_PRIORITIZE_FS_BASEGAME; + if (fs_restrict_dlfolder->integer) + { + // Don't allow config files from restricted download folder pk3s, because they could disable the download + // folder restrictions to unrestrict themselves + lookup_flags |= LOOKUPFLAG_NO_DOWNLOAD_FOLDER; + } + if (config_type == FS_CONFIGTYPE_SETTINGS) + { + // For autogen.cfg and autoexec.cfg - only load files on disk and from appropriate fs_mod_settings + // locations + lookup_flags |= (LOOKUPFLAG_SETTINGS_FILE | LOOKUPFLAG_DIRECT_SOURCE_ONLY); + } + + // NOTE: For now, just use the standard precedence for default.cfg. Since it gets overridden by the settings + // config files executed afterwards it usually shouldn't cause any problems even if it doesn't follow the + // fs_mod_settings policy. + + if (!quiet) + Com_Printf("execing %s\n", name); + + // Locate file + fs_refresh_auto(); + file = fs_general_lookup(name, lookup_flags, qfalse); + if (!file) + { + Com_Printf("couldn't exec %s - file not found\n", name); + fs_write_journal_data(0, 0); + return; + } + + // Load data + data = fs_read_data(file, 0, &size, "fs_execute_config_file"); + if (!data) + { + Com_Printf("couldn't exec %s - failed to read data\n", name); + fs_write_journal_data(0, 0); + return; + } + } + + fs_write_journal_data(data, size); + + Cbuf_ExecuteText(exec_type, data); + if (exec_type == EXEC_APPEND) + Cbuf_ExecuteText(EXEC_APPEND, "\n"); + fs_free_data(data); +} + +void *fs_load_game_dll( + const fsc_file_t *dll_file, intptr_t(QDECL **entryPoint)(int, ...), intptr_t(QDECL *systemcalls)(intptr_t, ...)) +{ + // Used by vm.c + // Returns dll handle, or null on error + char dll_info_string[FS_FILE_BUFFER_SIZE]; + const void *dll_path; + char *dll_path_string; + void *dll_handle; + FSC_ASSERT(dll_file); + + // Print the info message + fs_file_to_buffer(dll_file, dll_info_string, sizeof(dll_info_string), qtrue, qtrue, qtrue, qfalse); + Com_Printf("Attempting to load dll file at %s\n", dll_info_string); + + // Get dll path + if (dll_file->sourcetype != FSC_SOURCETYPE_DIRECT) + { + // Shouldn't happen + Com_Printf("Error: selected dll is not direct sourcetype\n"); + return 0; + } + dll_path = STACKPTR(((fsc_file_direct_t *)dll_file)->os_path_ptr); + dll_path_string = fsc_os_path_to_string(dll_path); + if (!dll_path_string) + { + // Generally shouldn't happen + Com_Printf("Error: failed to convert dll path\n"); + return 0; + } + + // Attemt to open the dll + dll_handle = Sys_LoadGameDll(dll_path_string, entryPoint, systemcalls); + if (!dll_handle) + { + Com_Printf("Error: failed to load game dll\n"); + } + fsc_free(dll_path_string); + return dll_handle; +} + +void FS_GetModDescription(const char *modDir, char *description, int descriptionLen) +{ + const char *descPath = va("%s/description.txt", modDir); + fileHandle_t descHandle; + int descLen = FS_SV_FOpenFileRead(descPath, &descHandle); + if (descLen > 0 && descHandle) + { + descLen = FS_Read(description, descriptionLen - 1, descHandle); + description[descLen] = 0; + } + if (descHandle) + fs_handle_close(descHandle); + if (descLen <= 0) + { + // Just use the mod name as the description + Q_strncpyz(description, modDir, descriptionLen); + } +} + +void FS_FilenameCompletion(const char *dir, const char *ext, qboolean stripExt, void (*callback)(const char *s), + qboolean allowNonPureFilesOnDisk) +{ + char **filenames; + int nfiles; + int i; + char filename[MAX_STRING_CHARS]; + + // Currently using the less restrictive FLISTFLAG_IGNORE_PURE_LIST when allowNonPureFilesOnDisk is + // false, since that's what's used for map completion, and we want to ignore the pure list there + filenames = FS_FlagListFilteredFiles(dir, ext, NULL, &nfiles, + allowNonPureFilesOnDisk ? FLISTFLAG_PURE_ALLOW_DIRECT_SOURCE : FLISTFLAG_IGNORE_PURE_LIST); + + for (i = 0; i < nfiles; i++) + { + Q_strncpyz(filename, filenames[i], MAX_STRING_CHARS); + + if (stripExt) + { + COM_StripExtension(filename, filename, sizeof(filename)); + } + + callback(filename); + } + FS_FreeFileList(filenames); +} + +// From old filesystem. Used in a couple of places. +qboolean FS_FilenameCompare(const char *s1, const char *s2) +{ + int c1, c2; + + do + { + c1 = *s1++; + c2 = *s2++; + + if (c1 >= 'a' && c1 <= 'z') + { + c1 -= ('a' - 'A'); + } + if (c2 >= 'a' && c2 <= 'z') + { + c2 -= ('a' - 'A'); + } + + if (c1 == '\\' || c1 == ':') + { + c1 = '/'; + } + if (c2 == '\\' || c2 == ':') + { + c2 = '/'; + } + + if (c1 != c2) + { + return qtrue; // strings not equal + } + } while (c1); + + return qfalse; // strings are equal +} + +void QDECL FS_Printf(fileHandle_t h, const char *fmt, ...) +{ + va_list argptr; + char msg[MAXPRINTMSG]; + + va_start(argptr, fmt); + Q_vsnprintf(msg, sizeof(msg), fmt, argptr); + va_end(argptr); + + FS_Write(msg, strlen(msg), h); +} + +void fs_comma_separated_list(const char **strings, int count, fsc_stream_t *output) +{ + // Writes array of strings to stream separated by comma (useful for debug print purposes) + // Ignores strings that are null or empty + // Writes "" if nothing was written + int i; + qboolean have_item = qfalse; + FSC_ASSERT(strings); + FSC_ASSERT(output); + fsc_stream_append_string(output, ""); + for (i = 0; i < count; ++i) + { + if (strings[i] && *strings[i]) + { + if (have_item) + fsc_stream_append_string(output, ", "); + fsc_stream_append_string(output, strings[i]); + have_item = qtrue; + } + } + if (!have_item) + fsc_stream_append_string(output, ""); +} + +qboolean FS_idPak(const char *pak, const char *base, int numPaks) +{ + int i; + + for (i = 0; i < numPaks; i++) + { + if (!FS_FilenameCompare(pak, va("%s/pak%d", base, i))) + { + break; + } + } + if (i < numPaks) + { + return qtrue; + } + return qfalse; +} + +void fs_sanitize_mod_dir(const char *source, char *target) +{ + // Sanitizes mod dir string. If mod dir is invalid it will be replaced with empty string. + // Target should be size FSC_MAX_MODDIR + char buffer[FSC_MAX_MODDIR]; + + // Copy to buffer before calling fs_generate_path, to allow overly long mod names to be truncated + // instead of the normal fs_generate_path behavior of generating an empty string on overflow + Q_strncpyz(buffer, source, sizeof(buffer)); + if (!fs_generate_path(buffer, 0, 0, 0, 0, 0, target, FSC_MAX_MODDIR)) + *target = 0; +} + +const char *fs_profile_to_string(FS_Profile profile) +{ + switch (profile) + { + case FS_PROFILE_1_1: + return "1.1"; + case FS_PROFILE_GPP: + return "GPP"; + case FS_PROFILE_1_3: + return "1.3"; + case FS_PROFILE_UNSET: + return "unset"; + } + return "invalid"; +} + +/* ******************************************************************************** */ +// VM Hash Verification +/* ******************************************************************************** */ + +qboolean calculate_file_sha256(const fsc_file_t *file, unsigned char *output) +{ + // Returns qtrue on success, qfalse otherwise + unsigned int size = 0; + char *data = fs_read_data(file, 0, &size, "calculate_file_sha256"); + if (!data) + { + Com_Memset(output, 0, 32); + return qfalse; + } + fsc_calculate_sha256(data, size, output); + fs_free_data(data); + return qtrue; +} + +qboolean fs_check_trusted_vm_file(const fsc_file_t *file) +{ + // Returns qtrue if file is trusted, qfalse otherwise + unsigned char sha[32]; + if (!calculate_file_sha256(file, sha)) + return qfalse; + return fs_check_trusted_vm_hash(sha); +} + +void sha256_to_stream(unsigned char *sha, fsc_stream_t *output) +{ + int i; + char buffer[4]; + for (i = 0; i < 32; ++i) + { + Com_sprintf(buffer, sizeof(buffer), "%02x", sha[i]); + fsc_stream_append_string(output, buffer); + } +} + +/* ******************************************************************************** */ +// Core Pak Verification +/* ******************************************************************************** */ + +// This section is used to verify the core (ID) paks on startup, and produce +// appropriate warnings or errors if they are out of place + +#ifndef STANDALONE +static const unsigned int core_hashes[] = { + 1566731103u, 298122907u, 412165236u, 2991495316u, 1197932710u, 4087071573u, 3709064859u, 908855077u, 977125798u}; + +static const unsigned int missionpack_hashes[] = {2430342401u, 511014160u, 2662638993u, 1438664554u}; + +static qboolean check_default_cfg_pk3(const char *mod, const char *filename, unsigned int hash) +{ + // Returns qtrue if there is a pk3 containing default.cfg with either the given name or hash + fsc_hashtable_iterator_t hti; + fsc_file_t *file; + + fsc_hashtable_open(&fs.files, fsc_string_hash("default", 0), &hti); + while ((file = (fsc_file_t *)STACKPTRN(fsc_hashtable_next(&hti)))) + { + const fsc_file_direct_t *source_pk3; + if (fs_file_disabled(file, FD_CHECK_FILE_ENABLED | FD_CHECK_READ_INACTIVE_MODS)) + continue; + if (file->sourcetype != FSC_SOURCETYPE_PK3) + continue; + if (Q_stricmp((const char *)STACKPTR(file->qp_name_ptr), "default")) + continue; + if (!file->qp_ext_ptr || Q_stricmp((const char *)STACKPTR(file->qp_ext_ptr), "cfg")) + continue; + if (file->qp_dir_ptr) + continue; + + source_pk3 = fsc_get_base_file(file, &fs); + if (source_pk3->pk3_hash == hash) + return qtrue; + if (mod && Q_stricmp(fsc_get_mod_dir((const fsc_file_t *)source_pk3, &fs), mod)) + continue; + if (!Q_stricmp((const char *)STACKPTR(source_pk3->f.qp_name_ptr), filename)) + return qtrue; + } + + return qfalse; +} + +typedef struct { + const fsc_file_direct_t *name_match; + const fsc_file_direct_t *hash_match; +} core_pak_state_t; + +static core_pak_state_t get_pak_state(const char *mod, const char *filename, unsigned int hash) +{ + // Locates name and hash matches for a given pak + const fsc_file_direct_t *name_match = 0; + fsc_hashtable_iterator_t hti; + fsc_pk3_hash_map_entry_t *entry; + const fsc_file_direct_t *file; + + fsc_hashtable_open(&fs.files, fsc_string_hash(filename, 0), &hti); + while ((file = (const fsc_file_direct_t *)STACKPTRN(fsc_hashtable_next(&hti)))) + { + if (fs_file_disabled((fsc_file_t *)file, FD_CHECK_FILE_ENABLED | FD_CHECK_READ_INACTIVE_MODS)) + continue; + if (file->f.sourcetype != FSC_SOURCETYPE_DIRECT) + continue; + if (Q_stricmp((const char *)STACKPTR(file->f.qp_name_ptr), filename)) + continue; + if (!file->f.qp_ext_ptr || Q_stricmp((const char *)STACKPTR(file->f.qp_ext_ptr), "pk3")) + continue; + if (file->f.qp_dir_ptr) + continue; + if (mod && Q_stricmp(fsc_get_mod_dir((fsc_file_t *)file, &fs), mod)) + continue; + if (file->pk3_hash == hash) + { + core_pak_state_t result = {file, file}; + return result; + } + name_match = file; + } + + fsc_hashtable_open(&fs.pk3_hash_lookup, hash, &hti); + while ((entry = (fsc_pk3_hash_map_entry_t *)STACKPTRN(fsc_hashtable_next(&hti)))) + { + file = (const fsc_file_direct_t *)STACKPTR(entry->pk3); + if (fs_file_disabled((fsc_file_t *)file, FD_CHECK_FILE_ENABLED | FD_CHECK_READ_INACTIVE_MODS)) + continue; + if ((file->pk3_hash == hash)) + { + core_pak_state_t result = {name_match, file}; + return result; + } + } + + { + core_pak_state_t result = {name_match, 0}; + return result; + } +} + +static void generate_pak_warnings( + const char *mod, const char *filename, core_pak_state_t *state, fsc_stream_t *warning_popup_stream) +{ + // Prints console warning messages and appends warning popup string for a given pak + if (state->hash_match) + { + if (!state->name_match) + { + char hash_match_buffer[FS_FILE_BUFFER_SIZE]; + fs_file_to_buffer((fsc_file_t *)state->hash_match, hash_match_buffer, sizeof(hash_match_buffer), qfalse, + qtrue, qfalse, qfalse); + Com_Printf("NOTE: %s/%s.pk3 is misnamed, found correct file at %s\n", mod, filename, hash_match_buffer); + } + else if (state->name_match != state->hash_match) + { + char hash_match_buffer[FS_FILE_BUFFER_SIZE]; + fs_file_to_buffer((fsc_file_t *)state->hash_match, hash_match_buffer, sizeof(hash_match_buffer), qfalse, + qtrue, qfalse, qfalse); + Com_Printf( + "WARNING: %s/%s.pk3 has incorrect hash, found correct file at %s\n", mod, filename, hash_match_buffer); + } + } + else + { + if (state->name_match) + { + Com_Printf("WARNING: %s/%s.pk3 has incorrect hash\n", mod, filename); + fsc_stream_append_string(warning_popup_stream, va("%s/%s.pk3: incorrect hash\n", mod, filename)); + } + else + { + Com_Printf("WARNING: %s/%s.pk3 not found\n", mod, filename); + fsc_stream_append_string(warning_popup_stream, va("%s/%s.pk3: not found\n", mod, filename)); + } + } +} + +void fs_check_core_paks(void) +{ + int i; + core_pak_state_t core_states[ARRAY_LEN(core_hashes)]; + core_pak_state_t missionpack_states[ARRAY_LEN(missionpack_hashes)]; + qboolean missionpack_installed = qfalse; // Any missionpack paks detected + char warning_popup_buffer[1024]; + fsc_stream_t warning_popup_stream = {warning_popup_buffer, 0, sizeof(warning_popup_buffer), qfalse}; + + // Generate pak states + for (i = 0; i < ARRAY_LEN(core_hashes); ++i) + { + core_states[i] = get_pak_state(BASEGAME, va("pak%i", i), core_hashes[i]); + } + for (i = 0; i < ARRAY_LEN(missionpack_hashes); ++i) + { + missionpack_states[i] = get_pak_state("missionpack", va("pak%i", i), missionpack_hashes[i]); + if (missionpack_states[i].name_match || missionpack_states[i].hash_match) + missionpack_installed = qtrue; + } + + // Check for standalone mode + if (Q_stricmp(com_basegame->string, BASEGAME)) + { + qboolean have_id_pak = qfalse; + for (i = 0; i < ARRAY_LEN(core_hashes); ++i) + if (core_states[i].hash_match) + have_id_pak = qtrue; + for (i = 0; i < ARRAY_LEN(missionpack_hashes); ++i) + if (missionpack_states[i].hash_match) + have_id_pak = qtrue; + if (!have_id_pak) + { + Com_Printf("Enabling standalone mode - no ID paks found\n"); + Cvar_Set("com_standalone", "1"); + return; + } + } + + // Print console warning messages and build warning popup string + for (i = 0; i < ARRAY_LEN(core_hashes); ++i) + { + generate_pak_warnings(BASEGAME, va("pak%i", i), &core_states[i], &warning_popup_stream); + } + if (missionpack_installed) + for (i = 0; i < ARRAY_LEN(missionpack_hashes); ++i) + { + generate_pak_warnings("missionpack", va("pak%i", i), &missionpack_states[i], &warning_popup_stream); + } + + // Check for missing default.cfg + if (!check_default_cfg_pk3(BASEGAME, "pak0", core_hashes[0])) + { + if (core_states[0].name_match || core_states[0].hash_match) + Com_Error(ERR_FATAL, BASEGAME + "/pak0.pk3 appears to be corrupt (missing default.cfg). Please recopy" + " this file from your Quake 3 CD or reinstall Quake 3. You may also try" + " deleting fscache.dat in case it has become corrupt"); + else + Com_Error(ERR_FATAL, BASEGAME + "/pak0.pk3 is missing. Please recopy" + " this file from your Quake 3 CD or reinstall Quake 3"); + } + +#ifndef DEDICATED + // If warning popup info was generated, display warning popup + if (warning_popup_stream.position) + { + dialogResult_t result = Sys_Dialog(DT_OK_CANCEL, + va("The following game files appear" + " to be missing or corrupt. You can try to run the game anyway, but you may" + " experience errors or problems connecting to remote servers.\n\n%s\n" + "You may need to reinstall Quake 3, the v1.32 patch, and/or team arena.", + warning_popup_buffer), + "File Warning"); + if (result == DR_CANCEL) + Sys_Quit(); + } +#endif +} +#endif + +#endif // NEW_FILESYSTEM diff --git a/src/filesystem/fs_reference.cpp b/src/filesystem/fs_reference.cpp new file mode 100644 index 000000000..9ec0b893d --- /dev/null +++ b/src/filesystem/fs_reference.cpp @@ -0,0 +1,1809 @@ +/* +=========================================================================== +Copyright (C) 1999-2005 Id Software, Inc. +Copyright (C) 2017 Noah Metzger (chomenor@gmail.com) + +This file is part of Quake III Arena source code. + +Quake III Arena source code is free software; you can redistribute it +and/or modify it under the terms of the GNU General Public License as +published by the Free Software Foundation; either version 2 of the License, +or (at your option) any later version. + +Quake III Arena source code is distributed in the hope that it will be +useful, but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with Quake III Arena source code; if not, write to the Free Software +Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA +=========================================================================== +*/ + +#ifdef NEW_FILESYSTEM +#include +#include +#include +#include +#include +#include +#include "fslocal.h" + +#define MAX_REFERENCE_SET_ENTRIES 2048 +#define MAX_PURE_CHECKSUM_CACHE 256 + +#define SYSTEMINFO_RESERVED_SIZE 256 + +#define MAX_DOWNLOAD_LIST_STRING 2048 +#define MAX_PURE_LIST_STRING BIG_INFO_STRING + +#define REF_DPRINTF(...) \ + { \ + if (fs_debug_references->integer) \ + FS_DPrintf(__VA_ARGS__); \ + } + +/* ******************************************************************************** */ +// Referenced Pak Tracking +/* ******************************************************************************** */ + +// The "reference_tracker" set is filled by logging game references to pk3 files +// It currently serves two purposes: +// 1) To generate the pure validation string when fs_full_pure_validation is 1 +// (Although I have never seen any server that requires this to connect) +// 2) As a component of the download pak list creation via '*referenced_paks' rule +// (Although in most cases it is redundant to other selector rules) +// Basically this section could probably be removed with no noticable effects in most +// situations, but is kept for now just to be on the safe side for compatibility purposes + +typedef struct { + fs_hashtable_entry_t hte; + fsc_file_direct_t *pak; +} reference_tracker_entry_t; + +static fs_hashtable_t reference_tracker; + +static qboolean reference_tracker_add(fsc_file_direct_t *pak) +{ + // Returns qtrue on success, qfalse if already inserted or maximum hit + fs_hashtable_iterator_t it = fs_hashtable_iterate(&reference_tracker, pak->pk3_hash, qfalse); + reference_tracker_entry_t *entry; + + if (reference_tracker.element_count >= MAX_REFERENCE_SET_ENTRIES) + return qfalse; + + while ((entry = (reference_tracker_entry_t *)fs_hashtable_next(&it))) + { + if (entry->pak->pk3_hash == pak->pk3_hash) + { + return qfalse; + } + } + + entry = (reference_tracker_entry_t *)S_Malloc(sizeof(reference_tracker_entry_t)); + entry->pak = pak; + fs_hashtable_insert(&reference_tracker, (fs_hashtable_entry_t *)entry, pak->pk3_hash); + return qtrue; +} + +void fs_register_reference(const fsc_file_t *file) +{ + // Adds the source pk3 of the given file to the current referenced paks set + if (file->sourcetype != FSC_SOURCETYPE_PK3) + return; + + // Don't register references for certain extensions + if (file->qp_ext_ptr) + { + int i; + static const char *special_extensions[] = {"shader", "txt", "cfg", "config", "bot", "arena", "menu"}; + if (file->qp_ext_ptr) + { + const char *extension = (const char *)STACKPTR(file->qp_ext_ptr); + for (i = 0; i < ARRAY_LEN(special_extensions); ++i) + { + if (!Q_stricmp(extension, special_extensions[i])) + return; + } + } + } + + // Don't register reference in certain special cases + if (!Q_stricmp((const char *)STACKPTR(file->qp_name_ptr), "qagame") && file->qp_ext_ptr && + !Q_stricmp((const char *)STACKPTR(file->qp_ext_ptr), "qvm") && file->qp_dir_ptr && + !Q_stricmp((const char *)STACKPTR(file->qp_dir_ptr), "vm")) + return; + if (file->qp_dir_ptr && !Q_stricmp((const char *)STACKPTR(file->qp_dir_ptr), "levelshots")) + return; + + // Initialize reference_tracker if it isn't already + if (!reference_tracker.bucket_count) + { + fs_hashtable_initialize(&reference_tracker, 32); + } + + // Add the reference + if (reference_tracker_add((fsc_file_direct_t *)fsc_get_base_file(file, &fs))) + { + if (fs_debug_references->integer) + { + char temp[FS_FILE_BUFFER_SIZE]; + fs_file_to_buffer(file, temp, sizeof(temp), qtrue, qtrue, qtrue, qfalse); + REF_DPRINTF("recording reference: %s\n", temp); + } + } +} + +void FS_ClearPakReferences(int flags) +{ + REF_DPRINTF("clearing referenced paks\n"); + fs_hashtable_reset(&reference_tracker, 0); +} + +static void reftracker_gen_sort_key(const fsc_file_t *file, fsc_stream_t *output) +{ + fs_generate_core_sort_key(file, output, qtrue); + fs_write_sort_filename(file, output); + fs_write_sort_value(fs_get_source_dir_id(file), output); +} + +static int reftracker_compare_file(const fsc_file_t *file1, const fsc_file_t *file2) +{ + char buffer1[1024]; + char buffer2[1024]; + fsc_stream_t stream1 = {buffer1, 0, sizeof(buffer1), qfalse}; + fsc_stream_t stream2 = {buffer2, 0, sizeof(buffer2), qfalse}; + reftracker_gen_sort_key(file1, &stream1); + reftracker_gen_sort_key(file2, &stream2); + return fsc_memcmp( + stream2.data, stream1.data, stream1.position < stream2.position ? stream1.position : stream2.position); +} + +static int reftracker_qsort(const void *e1, const void *e2) +{ + return reftracker_compare_file(*(const fsc_file_t **)e1, *(const fsc_file_t **)e2); +} + +static fsc_file_direct_t **generate_referenced_pak_list(int *count_out) +{ + // Result must be freed by Z_Free + // Result will not be valid if reference_tracker is reset + int count = 0; + fs_hashtable_iterator_t it = fs_hashtable_iterate(&reference_tracker, 0, qtrue); + const reference_tracker_entry_t *entry; + fsc_file_direct_t **reference_list = + (fsc_file_direct_t **)Z_Malloc(sizeof(*reference_list) * reference_tracker.element_count); + + // Generate reference list + while ((entry = (reference_tracker_entry_t *)fs_hashtable_next(&it))) + { + if (count >= reference_tracker.element_count) + Com_Error(ERR_FATAL, "generate_referenced_pak_list list overflowed"); + reference_list[count] = entry->pak; + ++count; + } + if (count != reference_tracker.element_count) + Com_Error(ERR_FATAL, "generate_referenced_pak_list list underflow"); + + // Sort reference list + qsort(reference_list, count, sizeof(*reference_list), reftracker_qsort); + + if (count_out) + *count_out = count; + return reference_list; +} + +const char *FS_ReferencedPakNames(void) +{ + // This is just used for a certain debug command now + static char buffer[1000]; + fsc_stream_t stream = {buffer, 0, sizeof(buffer), 0}; + int i; + int count = 0; + fsc_file_direct_t **reference_list = generate_referenced_pak_list(&count); + + *buffer = 0; + for (i = 0; i < count; ++i) + { + if (stream.position) + fsc_stream_append_string(&stream, " "); + fs_file_to_stream((const fsc_file_t *)reference_list[i], &stream, qfalse, qfalse, qfalse, qfalse); + } + + Z_Free(reference_list); + return buffer; +} + +/* ******************************************************************************** */ +// Pure Validation +/* ******************************************************************************** */ + +// This section is used to generate a pure validation string to pass the SV_VerifyPaks_f +// check when connecting to legacy pure servers. + +typedef struct { + const fsc_file_direct_t *pk3; + int data_size; + int pure_checksum; + int checksum_feed; +} pure_checksum_entry_t; + +static void get_pure_checksum_entry_callback(void *context, char *data, int size) +{ + pure_checksum_entry_t **entry = (pure_checksum_entry_t **)context; + *entry = (pure_checksum_entry_t *)fsc_malloc(sizeof(pure_checksum_entry_t) + size); + (*entry)->data_size = size; + Com_Memcpy((char *)&(*entry)->checksum_feed + 4, data, size); +} + +static pure_checksum_entry_t *get_pure_checksum_entry(const fsc_file_direct_t *pk3) +{ + pure_checksum_entry_t *entry = 0; + fsc_load_pk3(STACKPTR(pk3->os_path_ptr), &fs, 0, 0, get_pure_checksum_entry_callback, &entry); + if (entry) + entry->pk3 = pk3; + return entry; +} + +static void update_pure_checksum_entry(pure_checksum_entry_t *entry, int checksum_feed) +{ + entry->checksum_feed = LittleLong(checksum_feed); + entry->pure_checksum = fsc_block_checksum(&entry->checksum_feed, entry->data_size + 4); +} + +// Pure checksum cache + +typedef struct pure_checksum_node_s { + struct pure_checksum_node_s *next; + pure_checksum_entry_t *entry; + int rank; +} pure_checksum_node_t; + +static int pure_checksum_rank = 0; +static pure_checksum_node_t *pure_checksum_cache; + +static int get_pure_checksum_for_pk3(const fsc_file_direct_t *pk3, int checksum_feed) +{ + int entry_count = 0; + pure_checksum_node_t *node = pure_checksum_cache; + pure_checksum_node_t *deletion_node = 0; + + while (node) + { + if (node->entry && node->entry->pk3 == pk3) + { + // Use existing entry + if (node->entry->checksum_feed != checksum_feed) + { + update_pure_checksum_entry(node->entry, checksum_feed); + } + node->rank = ++pure_checksum_rank; + return node->entry->pure_checksum; + } + + if (!deletion_node || node->rank < deletion_node->rank) + deletion_node = node; + + ++entry_count; + node = node->next; + } + + // Prepare new node + if (entry_count >= MAX_PURE_CHECKSUM_CACHE) + { + node = deletion_node; + if (node->entry) + fsc_free(node->entry); + } + else + { + node = (pure_checksum_node_t *)S_Malloc(sizeof(*node)); + node->next = pure_checksum_cache; + pure_checksum_cache = node; + } + + // Create new entry + node->entry = get_pure_checksum_entry(pk3); + if (!node->entry) + return 0; + update_pure_checksum_entry(node->entry, checksum_feed); + node->rank = ++pure_checksum_rank; + return node->entry->pure_checksum; +} + +static int get_pure_checksum_for_file(const fsc_file_t *file, int checksum_feed) +{ + if (!file) + return 0; + if (file->sourcetype != FSC_SOURCETYPE_PK3) + return 0; + return get_pure_checksum_for_pk3(fsc_get_base_file(file, &fs), checksum_feed); +} + +static void add_referenced_pure_pk3s(fsc_stream_t *stream, fs_hashtable_t *reference_tracker) +{ + int i; + int count = 0; + fsc_file_direct_t **reference_list = generate_referenced_pak_list(&count); + char buffer[20]; + int lump_checksum = 0; + + // Process entries + for (i = 0; i < count; ++i) + { + int pure_checksum = get_pure_checksum_for_pk3(reference_list[i], checksum_feed); + + if (fs_debug_references->integer) + { + char temp[FS_FILE_BUFFER_SIZE]; + fs_file_to_buffer((fsc_file_t *)reference_list[i], temp, sizeof(temp), qtrue, qtrue, qtrue, qfalse); + REF_DPRINTF("adding pak to pure validation list: %s\n", temp); + } + + lump_checksum ^= pure_checksum; + Com_sprintf(buffer, sizeof(buffer), " %i", pure_checksum); + fsc_stream_append_string(stream, buffer); + } + + // Write final checksum + Com_sprintf(buffer, sizeof(buffer), " %i ", checksum_feed ^ lump_checksum ^ count); + fsc_stream_append_string(stream, buffer); + + Z_Free(reference_list); +} + +static void build_pure_validation_string(char *output, unsigned int output_size, fs_hashtable_t *reference_tracker) +{ + fsc_stream_t stream = {output, 0, output_size, 0}; + char buffer[50]; + int cgame_checksum = get_pure_checksum_for_file( + fs_general_lookup("vm/cgame.qvm", LOOKUPFLAG_IGNORE_CURRENT_MAP, qfalse), checksum_feed); + int ui_checksum = get_pure_checksum_for_file( + fs_general_lookup("vm/ui.qvm", LOOKUPFLAG_IGNORE_CURRENT_MAP, qfalse), checksum_feed); + + Com_sprintf(buffer, sizeof(buffer), "%i %i @", cgame_checksum, ui_checksum); + fsc_stream_append_string(&stream, buffer); + + if (fs_full_pure_validation->integer && connected_server_sv_pure != 2) + { + add_referenced_pure_pk3s(&stream, reference_tracker); + } + else + { + Com_sprintf(buffer, sizeof(buffer), " %i %i ", cgame_checksum, checksum_feed ^ cgame_checksum ^ 1); + fsc_stream_append_string(&stream, buffer); + } +} + +const char *FS_ReferencedPakPureChecksums(void) +{ + // Returns a space separated string containing the pure checksums of all referenced pk3 files. + // Servers with sv_pure set will get this string back from clients for pure validation + // The string has a specific order, "cgame ui @ ref1 ref2 ref3 ..." + static char buffer[1000]; + build_pure_validation_string(buffer, sizeof(buffer), &reference_tracker); + return buffer; +} + +/* ******************************************************************************** */ +// Shared Reference Structures +/* ******************************************************************************** */ + +// The reference query represents the input to the reference set generation functions. +typedef struct { + // Manifest string (from manifest cvars) + const char *manifest; + + // Enable certain special handling if query is for download list + qboolean download; + + // Tremulous - client profile for version-specific rules + clientProfile_t client_profile; +} reference_query_t; + +// Each reference list entry corresponds to one hash+name pair in the output pure/download +// list strings. +typedef struct { + // Primary characteristics + char mod_dir[FSC_MAX_MODDIR]; + char name[FSC_MAX_QPATH]; + unsigned int hash; + const fsc_file_direct_t *pak_file; // Optional (if null, represents hash-only entry) + + // For debug print purposes + char command_name[64]; // Name of the selector command that created this entry + int entry_id; // Numerical value assigned to entry to identify it in debug prints + + // Don't write to final string output + qboolean disabled; +} reference_list_entry_t; + +// Current state of a reference set / list / strings structure +typedef enum { REFSTATE_UNINITIALIZED, REFSTATE_OVERFLOWED, REFSTATE_VALID } reference_state_t; + +/* ******************************************************************************** */ +// Reference Set Generation +/* ******************************************************************************** */ + +// This section is used to create a reference set from a reference query. + +typedef struct { + // REFSTATE_VALID: Hashtable will be initialized and iterable + // REFSTATE_UNINITIALIZED / REFSTATE_OVERFLOWED: Hashtable not initialized + reference_state_t state; + fs_hashtable_t h; +} reference_set_t; + +typedef struct { + fs_hashtable_entry_t hte; + reference_list_entry_t l; + + // Misc sorting characteristics + int pak_file_name_match; + unsigned int cluster; // Indicates dash separated cluster (lower value is higher priority) + + // Sort key + char sort_key[FSC_MAX_MODDIR + FSC_MAX_QPATH + 32]; + unsigned int sort_key_length; +} reference_set_entry_t; + +typedef enum { RSPC_NONE, RSPC_CVAR_IMPORT } reference_set_pending_command_t; + +typedef enum { + RSBM_NONE, + RSBM_IGNORE, // Ignore the current entry + RSBM_BLOCK, // Add current entry to block list +} reference_set_block_mode_t; + +static const char *refset_block_mode_string(reference_set_block_mode_t mode) +{ + switch (mode) + { + case RSBM_NONE: + return "standard"; + case RSBM_IGNORE: + return "ignore"; + case RSBM_BLOCK: + return "block"; + } + return "unknown"; +} + +typedef struct { + // General state + const reference_query_t *query; + reference_set_t *reference_set; + pk3_list_t block_set; + int cluster; + qboolean overflowed; + + // Current command + reference_set_block_mode_t block_mode; + + // For debug prints + int entry_id_counter; + char command_name[64]; +} reference_set_work_t; + +static void refset_sanitize_string(const char *source, char *target, unsigned int size) +{ + // Sanitizes string to be suitable for output reference lists + // May write null string to target due to errors + char buffer[FSC_MAX_QPATH]; + char *buf_ptr = buffer; + if (size > sizeof(buffer)) + size = sizeof(buffer); + Q_strncpyz(buffer, source, size); + + // Underscore a couple characters that cause issues in ref strings but + // aren't handled by fs_generate_path + while (*buf_ptr) + { + if (*buf_ptr == ' ' || *buf_ptr == '@') + *buf_ptr = '_'; + ++buf_ptr; + } + + fs_generate_path(buffer, 0, 0, 0, 0, 0, target, size); +} + +static void refset_generate_entry(reference_set_work_t *rsw, const char *mod_dir, const char *name, unsigned int hash, + const fsc_file_direct_t *pak_file, reference_set_entry_t *target) +{ + // pak can be null; other parameters are required + fsc_stream_t sort_stream = {target->sort_key, 0, sizeof(target->sort_key), 0}; + + Com_Memset(target, 0, sizeof(*target)); + refset_sanitize_string(mod_dir, target->l.mod_dir, sizeof(target->l.mod_dir)); + refset_sanitize_string(name, target->l.name, sizeof(target->l.name)); + target->l.hash = hash; + target->l.pak_file = pak_file; + target->cluster = rsw->cluster; + target->l.entry_id = rsw->entry_id_counter++; + + // Write command name debug string + Q_strncpyz(target->l.command_name, rsw->command_name, sizeof(target->l.command_name)); + if (fsc_strlen(rsw->command_name) >= sizeof(target->l.command_name)) + { + strcpy(target->l.command_name + sizeof(target->l.command_name) - 4, "..."); + } + + // Determine pak_file_name_match, which is added to the sort key to handle special cases + // e.g. if a pk3 is specified in the download manifest with a specific hash, and multiple pk3s exist + // in the filesystem with that hash, this sort value attempts to prioritize the physical pk3 closer to + // the user-specified name to be used as the physical download source file + // 0 = no pak, 1 = no name match, 2 = case insensitive match, 3 = case sensitive match + if (pak_file) + { + const char *pak_mod = (const char *)STACKPTR(pak_file->qp_mod_ptr); + const char *pak_name = (const char *)STACKPTR(pak_file->f.qp_name_ptr); + if (!fsc_strcmp(mod_dir, pak_mod) && !fsc_strcmp(name, pak_name)) + target->pak_file_name_match = 3; + else if (!Q_stricmp(mod_dir, pak_mod) && !Q_stricmp(name, pak_name)) + target->pak_file_name_match = 2; + else + target->pak_file_name_match = 1; + } + + // Write sort key + { + FS_ModType mod_type = fs_get_mod_type(target->l.mod_dir); + unsigned int core_pak_priority = + mod_type < MODTYPE_OVERRIDE_DIRECTORY ? (unsigned int)core_pk3_position(hash) : 0; + + fs_write_sort_value(~target->cluster, &sort_stream); + fs_write_sort_value(mod_type >= MODTYPE_OVERRIDE_DIRECTORY ? (unsigned int)mod_type : 0, &sort_stream); + fs_write_sort_value(core_pak_priority, &sort_stream); + fs_write_sort_value((unsigned int)mod_type, &sort_stream); + fs_write_sort_string(target->l.mod_dir, &sort_stream, qfalse); + fs_write_sort_string(target->l.name, &sort_stream, qfalse); + fs_write_sort_value(target->pak_file_name_match, &sort_stream); + target->sort_key_length = sort_stream.position; + } +} + +static int refset_compare_entry(const reference_set_entry_t *e1, const reference_set_entry_t *e2) +{ + // Returns < 0 if e1 is higher precedence, > 0 if e2 is higher precedence + return -fsc_memcmp(e1->sort_key, e2->sort_key, + e1->sort_key_length < e2->sort_key_length ? e1->sort_key_length : e2->sort_key_length); +} + +static void refset_insert_entry( + reference_set_work_t *rsw, const char *mod_dir, const char *name, unsigned int hash, const fsc_file_direct_t *pak) +{ + // Inserts or updates reference entry into output reference set + reference_set_entry_t new_entry; + fs_hashtable_iterator_t it; + reference_set_entry_t *target_entry = 0; + + // Peform some mod dir patching for download list + if (rsw->query->download) + { + // Replace basemod with fs_basegame since downloads aren't supposed + // to go directly into basemod and clients may block it or have errors + if (!Q_stricmp(mod_dir, "basemod")) + { + REF_DPRINTF("[manifest processing] Replacing download mod directory 'basemod' with fs_basegame\n"); + mod_dir = fs_basegame->string; + } + + // Patch mod dir capitalization + if (!Q_stricmp(mod_dir, fs_basegame->string)) + mod_dir = fs_basegame->string; + if (!Q_stricmp(mod_dir, FS_GetCurrentGameDir())) + mod_dir = FS_GetCurrentGameDir(); + } + + // Generate new entry + refset_generate_entry(rsw, mod_dir, name, hash, pak, &new_entry); + + // Print entry contents + if (fs_debug_references->integer) + { + REF_DPRINTF("[manifest processing] Reference set entry created (mode '%s')\n", + refset_block_mode_string(rsw->block_mode)); + REF_DPRINTF(" entry id: %i\n", new_entry.l.entry_id); + REF_DPRINTF(" source rule: %s\n", new_entry.l.command_name); + REF_DPRINTF(" path: %s/%s\n", new_entry.l.mod_dir, new_entry.l.name); + REF_DPRINTF(" hash: %i\n", (int)new_entry.l.hash); + if (new_entry.l.pak_file) + { + char buffer[FS_FILE_BUFFER_SIZE]; + fs_file_to_buffer( + (const fsc_file_t *)new_entry.l.pak_file, buffer, sizeof(buffer), qtrue, qtrue, qtrue, qfalse); + REF_DPRINTF(" pak file: %s\n", buffer); + REF_DPRINTF(" pak file name match: %u\n", new_entry.pak_file_name_match); + } + else + { + REF_DPRINTF(" pak file: \n"); + } + REF_DPRINTF(" cluster: %i\n", new_entry.cluster); + } + + // Check for invalid attributes + if (!*new_entry.l.mod_dir || !*new_entry.l.name || !new_entry.l.hash) + { + REF_DPRINTF(" result: Skipping download list entry due to invalid mod, name, or hash\n"); + return; + } + +#ifndef STANDALONE + // Exclude paks that fail the ID pak check from download list because clients won't download + // them anyway and may throw an error + if (rsw->query->download) + { + char buffer[256]; + Com_sprintf(buffer, sizeof(buffer), "%s/%s", mod_dir, name); + if (FS_idPak(buffer, BASEGAME, FS_NODOWNLOAD_PAKS) || FS_idPak(buffer, BASETA, FS_NODOWNLOAD_PAKS_TEAMARENA)) + { + REF_DPRINTF(" result: Skipping download list entry due to ID pak name\n"); + return; + } + } +#endif + + // Check if hash is blocked + if (pk3_list_lookup(&rsw->block_set, hash)) + { + REF_DPRINTF(" result: Skipping entry due to hash in block list\n"); + return; + } + + // Process block modes + if (rsw->block_mode == RSBM_IGNORE) + { + REF_DPRINTF(" result: Skipping entry due to ignore mode\n"); + return; + } + if (rsw->block_mode == RSBM_BLOCK) + { + REF_DPRINTF(" result: Hash added to block list\n"); + pk3_list_insert(&rsw->block_set, hash); + return; + } + + // Look for existing entry with same hash + it = fs_hashtable_iterate(&rsw->reference_set->h, hash, qfalse); + while ((target_entry = (reference_set_entry_t *)fs_hashtable_next(&it))) + { + if (new_entry.l.hash == target_entry->l.hash) + { + // Found entry; check if new entry is higher priority + int compare_result = refset_compare_entry(&new_entry, target_entry); + if (fs_debug_references->integer) + { + if (compare_result >= 0) + { + REF_DPRINTF(" result: Duplicate hash - skipping entry due to existing %s precedence entry id %i\n", + compare_result > 0 ? "higher" : "equal", target_entry->l.entry_id); + } + else + { + REF_DPRINTF(" result: Duplicate hash - overwriting existing lower precedence entry id %i\n", + target_entry->l.entry_id); + } + } + if (compare_result < 0) + *target_entry = new_entry; + return; + } + } + + // Check for excess element count + if (rsw->reference_set->h.element_count >= MAX_REFERENCE_SET_ENTRIES) + { + REF_DPRINTF(" result: Skipping entry due to MAX_REFERENCE_SET_ENTRIES hit\n"); + rsw->overflowed = qtrue; + return; + } + + // Save the entry + REF_DPRINTF(" result: Added entry to reference set\n"); + target_entry = (reference_set_entry_t *)Z_Malloc(sizeof(*target_entry)); + *target_entry = new_entry; + fs_hashtable_insert(&rsw->reference_set->h, (fs_hashtable_entry_t *)target_entry, target_entry->l.hash); +} + +static void refset_insert_pak(reference_set_work_t *rsw, const fsc_file_direct_t *pak) +{ + // Add a particular pak file to the reference set + refset_insert_entry( + rsw, (const char *)STACKPTR(pak->qp_mod_ptr), (const char *)STACKPTR(pak->f.qp_name_ptr), pak->pk3_hash, pak); +} + +static void refset_add_referenced_paks(reference_set_work_t *rsw) +{ + // Add all current referenced paks to the reference set + fs_hashtable_iterator_t it = fs_hashtable_iterate(&reference_tracker, 0, qtrue); + reference_tracker_entry_t *entry; + while ((entry = (reference_tracker_entry_t *)fs_hashtable_next(&it))) + { + refset_insert_pak(rsw, entry->pak); + } +} + +static void refset_add_pak_containing_file(reference_set_work_t *rsw, const char *name) +{ + // Add the pak containing the specified file to the reference set + const fsc_file_t *file = + fs_general_lookup(name, LOOKUPFLAG_IGNORE_CURRENT_MAP | LOOKUPFLAG_PK3_SOURCE_ONLY, qfalse); + if (!file || file->sourcetype != FSC_SOURCETYPE_PK3) + { + return; + } + refset_insert_pak(rsw, fsc_get_base_file(file, &fs)); +} + +typedef enum { PAKCATEGORY_ACTIVE_MOD, PAKCATEGORY_BASEGAME, PAKCATEGORY_INACTIVE_MOD } pakcategory_t; + +static pakcategory_t refset_get_pak_category(const fsc_file_direct_t *pak) +{ + int mod_type = fs_get_mod_type(fsc_get_mod_dir((fsc_file_t *)pak, &fs)); + if (mod_type >= MODTYPE_CURRENT_MOD) + return PAKCATEGORY_ACTIVE_MOD; + if (mod_type > MODTYPE_INACTIVE) + return PAKCATEGORY_BASEGAME; + return PAKCATEGORY_INACTIVE_MOD; +} + +static void refset_add_paks_by_category(reference_set_work_t *rsw, pakcategory_t category) +{ + // Add all loaded paks in specified category to the pak set + int i; + fsc_hashtable_iterator_t hti; + fsc_pk3_hash_map_entry_t *hash_entry; + + for (i = 0; i < fs.pk3_hash_lookup.bucket_count; ++i) + { + fsc_hashtable_open(&fs.pk3_hash_lookup, i, &hti); + while ((hash_entry = (fsc_pk3_hash_map_entry_t *)STACKPTRN(fsc_hashtable_next(&hti)))) + { + fsc_file_direct_t *pk3 = (fsc_file_direct_t *)STACKPTR(hash_entry->pk3); + // The *inactivemod_paks rule explicitly follows the fs_read_inactive_mods setting in order for + // fs_read_inactive_mods to work in the expected way when using the default pure manifest + // Note: Pure list from a previous client session should be cleared at this point in the map load process, + // so the potential pure list check in FD_CHECK_READ_INACTIVE_MODS should not be a factor here. + if (fs_file_disabled((fsc_file_t *)pk3, FD_CHECK_FILE_ENABLED | FD_CHECK_READ_INACTIVE_MODS)) + continue; + if (refset_get_pak_category(pk3) != category) + continue; + refset_insert_pak(rsw, pk3); + } + } +} + +static unsigned int refset_string_to_hash(const char *string) +{ + // Converts a user-specified string (signed or unsigned) to hash value + // Returns 0 on error, hash otherwise + char test_buffer[16]; + if (*string == '-') + { + unsigned int hash = (unsigned int)atoi(string); + Com_sprintf(test_buffer, sizeof(test_buffer), "%i", (int)hash); + if (fsc_strcmp(string, test_buffer)) + return 0; + return hash; + } + else + { + unsigned int hash = strtoul(string, 0, 10); + Com_sprintf(test_buffer, sizeof(test_buffer), "%u", hash); + if (fsc_strcmp(string, test_buffer)) + return 0; + return hash; + } +} + +typedef struct { + char mod_dir[FSC_MAX_MODDIR]; + char name[FSC_MAX_QPATH]; + unsigned int hash; // 0 if hash not manually specified +} pak_specifier_t; + +static qboolean refset_parse_specifier(const char *command_name, const char *string, pak_specifier_t *output) +{ + // Converts specifier string to pak_specifier_t structure + // Returns qtrue on success, prints warning and returns qfalse on error + char buffer[FSC_MAX_MODDIR + FSC_MAX_QPATH]; + const char *hash_ptr = strchr(string, ':'); + const char *name_ptr = 0; + + if (hash_ptr) + { + // Copy section before colon to buffer + unsigned int length = (unsigned int)(hash_ptr - string); + if (length >= sizeof(buffer)) + length = sizeof(buffer) - 1; + fsc_memcpy(buffer, string, length); + buffer[length] = 0; + + // Acquire hash value + output->hash = refset_string_to_hash(hash_ptr + 1); + if (!output->hash) + { + Com_Printf("WARNING: Error reading hash for specifier '%s'\n", command_name); + return qfalse; + } + } + else + { + Q_strncpyz(buffer, string, sizeof(buffer)); + output->hash = 0; + } + + fsc_get_leading_directory(buffer, output->mod_dir, sizeof(output->mod_dir), &name_ptr); + if (!*output->mod_dir) + { + Com_Printf("WARNING: Error reading mod directory for specifier '%s'\n", command_name); + return qfalse; + } + if (!name_ptr || !*name_ptr || strchr(name_ptr, '/') || strchr(name_ptr, '\\')) + { + Com_Printf("WARNING: Error reading pk3 name for specifier '%s'\n", command_name); + return qfalse; + } + Q_strncpyz(output->name, name_ptr, sizeof(output->name)); + + return qtrue; +} + +static void refset_process_specifier_by_name(reference_set_work_t *rsw, const char *string) +{ + // Process a pak specifier in format / + pak_specifier_t specifier; + int count = 0; + fsc_hashtable_iterator_t hti; + const fsc_file_direct_t *file; + + if (!refset_parse_specifier(rsw->command_name, string, &specifier)) + return; + FSC_ASSERT(!specifier.hash); + + // Search for pk3s matching name + fsc_hashtable_open(&fs.files, fsc_string_hash(specifier.name, 0), &hti); + while ((file = (const fsc_file_direct_t *)STACKPTRN(fsc_hashtable_next(&hti)))) + { + if (file->f.sourcetype != FSC_SOURCETYPE_DIRECT) + continue; + if (!file->pk3_hash) + continue; + if (fs_file_disabled((const fsc_file_t *)file, FD_CHECK_FILE_ENABLED)) + continue; + if (Q_stricmp((const char *)STACKPTR(file->f.qp_name_ptr), specifier.name)) + continue; + if (Q_stricmp(fsc_get_mod_dir((const fsc_file_t *)file, &fs), specifier.mod_dir)) + continue; + refset_insert_entry(rsw, specifier.mod_dir, specifier.name, file->pk3_hash, file); + ++count; + } + + if (count == 0) + Com_Printf("WARNING: Specifier '%s' failed to match any pk3s.\n", rsw->command_name); + if (count > 1) + Com_Printf("WARNING: Specifier '%s' matched multiple pk3s.\n", rsw->command_name); +} + +static void refset_process_specifier_by_hash(reference_set_work_t *rsw, const char *string) +{ + // Process a pak specifier in format /: + pak_specifier_t specifier; + int count = 0; + fsc_hashtable_iterator_t hti; + fsc_pk3_hash_map_entry_t *entry; + + if (!refset_parse_specifier(rsw->command_name, string, &specifier)) + return; + FSC_ASSERT(specifier.hash); + + // Search for physical pk3s matching hash + fsc_hashtable_open(&fs.pk3_hash_lookup, specifier.hash, &hti); + while ((entry = (fsc_pk3_hash_map_entry_t *)STACKPTRN(fsc_hashtable_next(&hti)))) + { + const fsc_file_direct_t *file = (const fsc_file_direct_t *)STACKPTR(entry->pk3); + if (fs_file_disabled((fsc_file_t *)file, FD_CHECK_FILE_ENABLED)) + continue; + if (file->pk3_hash != specifier.hash) + continue; + refset_insert_entry(rsw, specifier.mod_dir, specifier.name, specifier.hash, file); + ++count; + } + + // If no actual pak was found, create a hash-only entry + if (!count) + { + refset_insert_entry(rsw, specifier.mod_dir, specifier.name, specifier.hash, 0); + ++count; + } +} + +static qboolean refset_pattern_match(const char *string, const char *pattern) +{ + // Returns qtrue if string matches pattern containing '*' and '?' wildcards + while (1) + { + if (*pattern == '*') + { + // Skip asterisks; auto match if no pattern remaining + while (*pattern == '*') + ++pattern; + if (!*pattern) + return qtrue; + + // Read string looking for match with remaining pattern + while (*string) + { + if (*string == *pattern || *pattern == '?') + { + if (refset_pattern_match(string + 1, pattern + 1)) + return qtrue; + } + ++string; + } + + // Leftover pattern with no match + return qfalse; + } + + // Check for end of string cases + if (!*pattern) + { + if (!*string) + return qtrue; + return qfalse; + } + if (!*string) + return qfalse; + + // Check for character discrepancy + if (*pattern != *string && *pattern != '?' && *pattern != *string) + return qfalse; + + // Advance strings + ++pattern; + ++string; + } +} + +static void refset_process_specifier_by_wildcard(reference_set_work_t *rsw, const char *string) +{ + // Process a pak specifier in format / containing wildcard characters + int count = 0; + unsigned int i; + fsc_hashtable_iterator_t hti; + fsc_pk3_hash_map_entry_t *entry; + char specifier_buffer[FSC_MAX_MODDIR + FSC_MAX_QPATH]; + char file_buffer[FSC_MAX_MODDIR + FSC_MAX_QPATH]; + char *z = specifier_buffer; + + Q_strncpyz(specifier_buffer, string, sizeof(specifier_buffer)); + Q_strlwr(specifier_buffer); + while (*z) + { + if (*z == '\\') + *z = '/'; + ++z; + } + + // Iterate all pk3s in filesystem for potential matches + for (i = 0; i < fs.pk3_hash_lookup.bucket_count; ++i) + { + fsc_hashtable_open(&fs.pk3_hash_lookup, i, &hti); + while ((entry = (fsc_pk3_hash_map_entry_t *)STACKPTRN(fsc_hashtable_next(&hti)))) + { + const fsc_file_direct_t *file = (const fsc_file_direct_t *)STACKPTR(entry->pk3); + const char *mod_dir = fsc_get_mod_dir((fsc_file_t *)file, &fs); + const char *name = (const char *)STACKPTR(file->f.qp_name_ptr); + if (fs_file_disabled((fsc_file_t *)file, FD_CHECK_FILE_ENABLED)) + continue; + + // Check pattern match + Com_sprintf(file_buffer, sizeof(file_buffer), "%s/%s", mod_dir, name); + Q_strlwr(file_buffer); + if (!refset_pattern_match(file_buffer, specifier_buffer)) + continue; + + // Add pk3 to reference set + refset_insert_entry(rsw, mod_dir, name, file->pk3_hash, file); + ++count; + } + } + + if (count == 0) + Com_Printf("WARNING: Specifier '%s' failed to match any pk3s.\n", rsw->command_name); +} + +static void refset_process_specifier(reference_set_work_t *rsw, const char *string) +{ + // Process pk3 specifier of any supported type (mod/name, mod/name:hash, wildcard) + if (strchr(string, '*') || strchr(string, '?')) + { + refset_process_specifier_by_wildcard(rsw, string); + } + else if (strchr(string, ':')) + { + refset_process_specifier_by_hash(rsw, string); + } + else + { + refset_process_specifier_by_name(rsw, string); + } +} + +static bool refset_check_version_selector(std::string version_selector, clientProfile_t client_profile) +{ + // Tremulous - returns true if version selector string matches client profile, false otherwise + std::stringstream stream(version_selector); + std::string token; + + while (std::getline(stream, token, ',')) + { + bool match = false; + if (!Q_stricmp(token.c_str(), "v1_1")) + match = (client_profile == CLIENT_PROFILE_1_1); + else if (!Q_stricmp(token.c_str(), "v1_1+")) + match = (client_profile >= CLIENT_PROFILE_1_1); + else if (!Q_stricmp(token.c_str(), "v1_1-")) + match = (client_profile <= CLIENT_PROFILE_1_1); + else if (!Q_stricmp(token.c_str(), "gpp")) + match = (client_profile == CLIENT_PROFILE_GPP); + else if (!Q_stricmp(token.c_str(), "gpp+")) + match = (client_profile >= CLIENT_PROFILE_GPP); + else if (!Q_stricmp(token.c_str(), "gpp-")) + match = (client_profile <= CLIENT_PROFILE_GPP); + else if (!Q_stricmp(token.c_str(), "v1_3")) + match = (client_profile == CLIENT_PROFILE_1_3); + else if (!Q_stricmp(token.c_str(), "v1_3+")) + match = (client_profile >= CLIENT_PROFILE_1_3); + else if (!Q_stricmp(token.c_str(), "v1_3-")) + match = (client_profile <= CLIENT_PROFILE_1_3); + else + { + Com_Printf("WARNING: Version selector '%s' contains unrecognized token '%s'\n", version_selector.c_str(), + token.c_str()); + } + if (match) + return true; + } + return false; +} +static void refset_process_manifest(reference_set_work_t *rsw, const char *string, int recursion_count) +{ + while (1) + { + const char *token = COM_ParseExt((char **)&string, qfalse); + if (!*token) + break; + + // Process special commands + if (!Q_stricmp(token, "&cvar_import")) + { + // Static buffer from COM_ParseExt will be overwritten, so copy out cvar name + char cvar_name[256]; + token = COM_ParseExt((char **)&string, qfalse); + Q_strncpyz(cvar_name, token, sizeof(cvar_name)); + + if (recursion_count >= 128) + { + Com_Error(ERR_DROP, "Recursive overflow processing pk3 manifest"); + } + REF_DPRINTF("[manifest processing] Entering import cvar '%s'\n", cvar_name); + refset_process_manifest(rsw, Cvar_VariableString(cvar_name), recursion_count + 1); + REF_DPRINTF("[manifest processing] Leaving import cvar '%s'\n", cvar_name); + continue; + } + else if (!Q_stricmp(token, "&select_version")) + { + token = COM_ParseExt((char **)&string, qfalse); + bool match = refset_check_version_selector(token, rsw->query->client_profile); + if (match) + { + REF_DPRINTF("[manifest processing] 'select_version' command matched version - setting standard mode\n"); + rsw->block_mode = RSBM_NONE; + } + else + { + REF_DPRINTF( + "[manifest processing] 'select_version' command did not match version - setting ignore mode\n"); + rsw->block_mode = RSBM_IGNORE; + } + continue; + } + else if (!Q_stricmp(token, "&allow_version")) + { + token = COM_ParseExt((char **)&string, qfalse); + bool match = refset_check_version_selector(token, rsw->query->client_profile); + if (match) + { + REF_DPRINTF("[manifest processing] 'allow_version' command matched version - setting ignore mode\n"); + rsw->block_mode = RSBM_IGNORE; + } + else + { + REF_DPRINTF( + "[manifest processing] 'allow_version' command did not match version - setting block mode\n"); + rsw->block_mode = RSBM_BLOCK; + } + continue; + } + else if (!Q_stricmp(token, "&block_version")) + { + token = COM_ParseExt((char **)&string, qfalse); + bool match = refset_check_version_selector(token, rsw->query->client_profile); + if (match) + { + REF_DPRINTF("[manifest processing] 'block_version' command matched version - setting block mode\n"); + rsw->block_mode = RSBM_BLOCK; + } + else + { + REF_DPRINTF( + "[manifest processing] 'block_version' command did not match version - setting ignore mode\n"); + rsw->block_mode = RSBM_IGNORE; + } + continue; + } + else if (!Q_stricmp(token, "&block")) + { + REF_DPRINTF("[manifest processing] Blocking next selector due to 'block' command\n"); + rsw->block_mode = RSBM_BLOCK; + continue; + } + else if (!Q_stricmp(token, "&block_reset")) + { + REF_DPRINTF("[manifest processing] Resetting blocked hash set.\n"); + pk3_list_free(&rsw->block_set); + pk3_list_initialize(&rsw->block_set, 64); + continue; + } + else if (!Q_stricmp(token, "-")) + { + ++rsw->cluster; + continue; + } + + // Process selector commands + Q_strncpyz(rsw->command_name, token, sizeof(rsw->command_name)); + REF_DPRINTF("[manifest processing] Processing selector '%s'\n", rsw->command_name); + Q_strncpyz(rsw->command_name, token, sizeof(rsw->command_name)); + if (!Q_stricmp(token, "#mod_paks")) + { + refset_add_paks_by_category(rsw, PAKCATEGORY_ACTIVE_MOD); + } + else if (!Q_stricmp(token, "#base_paks")) + { + refset_add_paks_by_category(rsw, PAKCATEGORY_BASEGAME); + } + else if (!Q_stricmp(token, "#inactivemod_paks")) + { + refset_add_paks_by_category(rsw, PAKCATEGORY_INACTIVE_MOD); + } + else if (!Q_stricmp(token, "#referenced_paks")) + { + refset_add_referenced_paks(rsw); + } + else if (!Q_stricmp(token, "#currentmap_pak")) + { + refset_add_pak_containing_file(rsw, va("maps/%s.bsp", Cvar_VariableString("mapname"))); + } + else if (!Q_stricmp(token, "#cgame_pak")) + { + refset_add_pak_containing_file(rsw, "vm/cgame.qvm"); + } + else if (!Q_stricmp(token, "#ui_pak")) + { + refset_add_pak_containing_file(rsw, "vm/ui.qvm"); + } + else if (*token == '#' || *token == '&') + { + Com_Printf("WARNING: Unrecognized manifest selector '%s'\n", token); + } + else + { + refset_process_specifier(rsw, token); + } + + // Reset single-use modifiers + rsw->block_mode = RSBM_NONE; + } +} + +static reference_set_t refset_uninitialized(void) +{ + reference_set_t result = {REFSTATE_UNINITIALIZED}; + return result; +} + +static reference_set_t refset_generate(const reference_query_t *query) +{ + // Generates reference set for given query + // Result must be freed by refset_free + reference_set_t output = refset_uninitialized(); + reference_set_work_t rsw; + + // Initialize output + output.state = REFSTATE_VALID; + fs_hashtable_initialize(&output.h, MAX_REFERENCE_SET_ENTRIES); + + // Initialize rsw + Com_Memset(&rsw, 0, sizeof(rsw)); + rsw.query = query; + rsw.reference_set = &output; + pk3_list_initialize(&rsw.block_set, 64); + + // Invoke manifest processing + refset_process_manifest(&rsw, query->manifest, 0); + + // Free rsw + pk3_list_free(&rsw.block_set); + + if (rsw.overflowed) + { + // Clear structure in case of overflow + fs_hashtable_free(&output.h, 0); + output = refset_uninitialized(); + output.state = REFSTATE_OVERFLOWED; + } + + return output; +} + +static void refset_free(reference_set_t *reference_set) +{ + if (reference_set->state == REFSTATE_VALID) + { + fs_hashtable_free(&reference_set->h, 0); + } +} + +/* ******************************************************************************** */ +// Reference List Generation +/* ******************************************************************************** */ + +// This section is used to create a reference list from a reference set. + +typedef struct { + // REFSTATE_VALID: Entries will be valid pointer, entry count >= 0 + // REFSTATE_UNINITIALIZED / REFSTATE_OVERFLOWED: Entries not valid pointer, entry count == 0 + reference_state_t state; + reference_list_entry_t *entries; + int entry_count; +} reference_list_t; + +static int reflist_sort_function(const void *e1, const void *e2) +{ + return refset_compare_entry(*(const reference_set_entry_t **)e1, *(const reference_set_entry_t **)e2); +} + +static reference_list_t reflist_uninitialized(void) +{ + reference_list_t result = {REFSTATE_UNINITIALIZED}; + return result; +} + +static reference_list_t reflist_generate(const reference_set_t *reference_set) +{ + // Converts reference set to reference list + // Result must be freed by reflist_free + reference_list_t reference_list = reflist_uninitialized(); + + if (reference_set->state == REFSTATE_VALID) + { + // Generate temp entries + int i; + int count = 0; + reference_set_entry_t *entry; + fs_hashtable_iterator_t it = fs_hashtable_iterate((fs_hashtable_t *)&reference_set->h, 0, qtrue); + reference_set_entry_t *temp_entries[MAX_REFERENCE_SET_ENTRIES]; + FSC_ASSERT(reference_set->h.element_count <= MAX_REFERENCE_SET_ENTRIES); + while ((entry = (reference_set_entry_t *)fs_hashtable_next(&it))) + { + FSC_ASSERT(count < reference_set->h.element_count); + temp_entries[count] = entry; + ++count; + } + FSC_ASSERT(count == reference_set->h.element_count); + + // Sort temp entries + qsort(temp_entries, count, sizeof(*temp_entries), reflist_sort_function); + + // Initialize reference list + reference_list.state = REFSTATE_VALID; + reference_list.entries = (reference_list_entry_t *)Z_Malloc(sizeof(*reference_list.entries) * count); + reference_list.entry_count = count; + + // Copy reference list entries + for (i = 0; i < count; ++i) + { + reference_list.entries[i] = temp_entries[i]->l; + } + } + + else + { + // Just leave the list uninitialized + reference_list.state = reference_set->state; + } + + return reference_list; +} + +static void reflist_free(reference_list_t *reference_list) +{ + if (reference_list->state == REFSTATE_VALID) + { + Z_Free(reference_list->entries); + } +} + +/* ******************************************************************************** */ +// Reference String Generation +/* ******************************************************************************** */ + +// This section is used to create reference strings from a reference list. + +typedef struct { + // REFSTATE_VALID: String will be pointer in Z_Malloc, length >= 0 + // REFSTATE_UNINITIALIZED / REFSTATE_OVERFLOWED: String equals static "", length == 0 + reference_state_t state; + char *string; + unsigned int length; // strlen(string) +} reference_substring_t; + +typedef struct { + reference_substring_t name; + reference_substring_t hash; +} reference_strings_t; + +static reference_substring_t refstrings_generate_substring(fsc_stream_t *source) +{ + // Convert source stream to reference string structure + reference_substring_t output = {REFSTATE_UNINITIALIZED, (char *)"", 0}; + if (source->overflowed) + output.state = REFSTATE_OVERFLOWED; + if (source->position && !source->overflowed) + { + output.state = REFSTATE_VALID; + output.length = source->position; + output.string = (char *)Z_Malloc(output.length + 1); + fsc_memcpy(output.string, source->data, output.length); + output.string[output.length] = 0; + } + return output; +} + +static void refstrings_free_substring(reference_substring_t *substring) +{ + if (substring->state == REFSTATE_VALID) + { + Z_Free(substring->string); + } +} + +static reference_strings_t refstrings_uninitialized(void) +{ + reference_strings_t result = {{REFSTATE_UNINITIALIZED, (char *)"", 0}, {REFSTATE_UNINITIALIZED, (char *)"", 0}}; + return result; +} + +static reference_strings_t refstrings_generate(reference_list_t *reference_list, unsigned int max_length) +{ + // Result must be freed by refstrings_free + reference_strings_t output = refstrings_uninitialized(); + + if (reference_list->state != REFSTATE_VALID) + { + if (reference_list->state == REFSTATE_OVERFLOWED) + { + // Copy overflowed state to string outputs + output.hash.state = output.name.state = REFSTATE_OVERFLOWED; + } + } + + else + { + int i; + char buffer[512]; + fsc_stream_t name_stream = {(char *)Z_Malloc(max_length), 0, max_length, 0}; + fsc_stream_t hash_stream = {(char *)Z_Malloc(max_length), 0, max_length, 0}; + + // Generate strings + for (i = 0; i < reference_list->entry_count; ++i) + { + const reference_list_entry_t *entry = &reference_list->entries[i]; + if (entry->disabled) + continue; + + Com_sprintf(buffer, sizeof(buffer), "%i", (int)entry->hash); + if (hash_stream.position) + fsc_stream_append_string(&hash_stream, " "); + fsc_stream_append_string(&hash_stream, buffer); + + Com_sprintf(buffer, sizeof(buffer), "%s/%s", entry->mod_dir, entry->name); + if (name_stream.position) + fsc_stream_append_string(&name_stream, " "); + fsc_stream_append_string(&name_stream, buffer); + } + + // Transfer strings to output structure + output.hash = refstrings_generate_substring(&hash_stream); + Z_Free(hash_stream.data); + output.name = refstrings_generate_substring(&name_stream); + Z_Free(name_stream.data); + } + + return output; +} + +static void refstrings_free(reference_strings_t *reference_strings) +{ + refstrings_free_substring(&reference_strings->hash); + refstrings_free_substring(&reference_strings->name); +} + +/* ******************************************************************************** */ +// Download Map Handling +/* ******************************************************************************** */ + +// The download map is used to match client download requests to the actual file +// on the server, since the download list name may not match the server filename + +class FS_DownloadMap { + std::map download_map; + +public: + FS_DownloadMap(const reference_list_t *reference_list) + { + char buffer[512]; + for (int i = 0; i < reference_list->entry_count; ++i) + { + const reference_list_entry_t *entry = &reference_list->entries[i]; + if (entry->disabled) + continue; + if (!entry->pak_file) + continue; + Com_sprintf(buffer, sizeof(buffer), "%s/%s.pk3", entry->mod_dir, entry->name); + Q_strlwr(buffer); + download_map.emplace(buffer, entry->pak_file); + } + } + + fileHandle_t open_download_pak(const char *path, unsigned int *size_out) + { + const fsc_file_direct_t *pak_file = 0; + char buffer[512]; + Q_strncpyz(buffer, path, sizeof(buffer)); + Q_strlwr(buffer); + try + { + pak_file = download_map.at(buffer); + } + catch (const std::out_of_range &oor) + { + } + + if (pak_file) + return fs_direct_read_handle_open((fsc_file_t *)pak_file, 0, size_out); + return 0; + } +}; + +class FS_DownloadMapShared { + std::shared_ptr shared_map; + +public: + void reset() { shared_map.reset(); } + + void load(const reference_list_t *reference_list) { shared_map = std::make_shared(reference_list); } + + fileHandle_t open_download_pak(const char *path, unsigned int *size_out) + { + if (shared_map) + return shared_map->open_download_pak(path, size_out); + return 0; + } +}; + +/* ******************************************************************************** */ +// Download / Pure List Generation +/* ******************************************************************************** */ + +static FS_DownloadMapShared current_download_maps[CLIENT_PROFILE_COUNT]; + +FS_ReferenceProfile::FS_ReferenceProfile(clientProfile_t profile) +{ + // Defines reference-related constants for a particular client profile + std::string cvar_suffix; + switch (profile) + { + case CLIENT_PROFILE_1_1: + profile_description = "v1.1"; + cvar_suffix = "_v1_1"; + break; + case CLIENT_PROFILE_GPP: + profile_description = "gpp"; + cvar_suffix = "_gpp"; + break; + case CLIENT_PROFILE_1_3: + profile_description = "v1.3"; + cvar_suffix = ""; + break; + } + + download_hashes_cvar = "sv_referencedPaks" + cvar_suffix; + download_names_cvar = "sv_referencedPakNames" + cvar_suffix; + pure_hashes_cvar = "sv_paks" + cvar_suffix; + pure_names_cvar = "sv_pakNames" + cvar_suffix; +} + +static reference_list_t reference_list_from_query(const reference_query_t *query) +{ + reference_set_t reference_set = refset_generate(query); + reference_list_t reference_list = reflist_generate(&reference_set); + refset_free(&reference_set); + return reference_list; +} + +static qboolean hash_in_reference_list(reference_list_t *reference_list, unsigned int hash) +{ + int i; + for (i = 0; i < reference_list->entry_count; ++i) + { + if (reference_list->entries[i].hash == hash) + return qtrue; + } + return qfalse; +} + +static void fs_generate_reference_lists_profile(clientProfile_t client_profile) +{ + // Generate download and pure lists for a particular client profile and set appropriate cvars + int i, j; + FS_ReferenceProfile reference_profile(client_profile); + reference_query_t download_query = {fs_download_manifest->string, qtrue, client_profile}; + reference_query_t pure_query = {fs_pure_manifest->string, qfalse, client_profile}; + reference_list_t download_list = reflist_uninitialized(); + reference_list_t pure_list = reflist_uninitialized(); + reference_strings_t download_strings = refstrings_uninitialized(); + reference_strings_t pure_strings = refstrings_uninitialized(); + qboolean download_valid = qtrue; + qboolean pure_valid = qfalse; + qboolean pure_names_valid = qfalse; + + // Generate download list + Com_Printf("Generating download list [%s]...\n", reference_profile.profile_description.c_str()); + fs_debug_indent_start(); + download_list = reference_list_from_query(&download_query); + fs_debug_indent_stop(); + Com_Printf("%i paks listed\n", download_list.entry_count); + + // Verify download list + for (i = 0; i < download_list.entry_count; ++i) + { + int allowDownload = Cvar_VariableIntegerValue("sv_allowDownload"); + reference_list_entry_t *entry = &download_list.entries[i]; + + // Check for entry with duplicate filename + for (j = 0; j < i; ++j) + { + reference_list_entry_t *entry2 = &download_list.entries[j]; + if (entry2->disabled) + continue; + if (!Q_stricmp(entry->mod_dir, entry2->mod_dir) && !Q_stricmp(entry->name, entry2->name)) + { + break; + } + } + if (j < i) + { + Com_Printf( + "WARNING: Skipping download list pak '%s/%s' with same filename but" + " different hash as another entry.\n", + entry->mod_dir, entry->name); + entry->disabled = qtrue; + continue; + } + + // Print warning if file is physically unavailable + if (!entry->pak_file && allowDownload && !(allowDownload & DLF_NO_UDP)) + { + Com_Printf( + "WARNING: Download list pak '%s/%s' from command '%s' was not found on the server." + " Attempts to download this file via UDP will result in an error.\n", + entry->mod_dir, entry->name, entry->command_name); + } + + // Print warning if pak is from an inactive mod dir + if (fs_get_mod_type(entry->mod_dir) <= MODTYPE_INACTIVE) + { + Com_Printf( + "WARNING: Download list pak '%s/%s' from command '%s' is from an inactive mod dir." + " This can cause problems for some clients. Consider moving this file or changing the" + " active mod to include it.\n", + entry->mod_dir, entry->name, entry->command_name); + } + } + + // Generate download strings + download_strings = refstrings_generate(&download_list, MAX_DOWNLOAD_LIST_STRING); + + // Check for download list overflow + if (download_strings.hash.state == REFSTATE_OVERFLOWED || download_strings.name.state == REFSTATE_OVERFLOWED) + { + Com_Printf("WARNING: Download list overflowed\n"); + download_valid = qfalse; + } + + if (Cvar_VariableIntegerValue("sv_pure")) + { + int systeminfo_base_length = fsc_strlen(Cvar_InfoString_Big(CVAR_SYSTEMINFO)); + int download_base_length = download_valid ? download_strings.name.length + download_strings.hash.length : 0; + pure_valid = pure_names_valid = qtrue; + + // Generate pure list + Com_Printf("Generating pure list [%s]...\n", reference_profile.profile_description.c_str()); + fs_debug_indent_start(); + pure_list = reference_list_from_query(&pure_query); + fs_debug_indent_stop(); + Com_Printf("%i paks listed\n", pure_list.entry_count); + + // Generate pure strings + pure_strings = refstrings_generate(&pure_list, MAX_PURE_LIST_STRING); + + // Check for pure list hash overflow + if (pure_strings.hash.state == REFSTATE_OVERFLOWED) + { + Com_Printf( + "WARNING: Setting empty pure list due to pure list overflow. Remove some" + " paks from the server or adjust the pure manifest if you want to use sv_pure.\n"); + pure_valid = pure_names_valid = qfalse; + } + + // Check for empty pure list + if (pure_valid && pure_list.entry_count == 0) + { + Com_Printf("WARNING: Empty pure list.\n"); + pure_valid = pure_names_valid = qfalse; + } + + // Check for pure list hash systeminfo overflow + if (pure_valid && + systeminfo_base_length + download_base_length + pure_strings.hash.length + SYSTEMINFO_RESERVED_SIZE >= + BIG_INFO_STRING) + { + Com_Printf( + "WARNING: Setting empty pure list due to systeminfo overflow. Remove some" + " paks from the server or adjust the pure manifest if you want to use sv_pure.\n"); + pure_valid = pure_names_valid = qfalse; + } + + // Check for pure list names output overflow + if (pure_names_valid && pure_strings.name.state == REFSTATE_OVERFLOWED) + { + Com_Printf("NOTE: Not writing optional sv_pakNames value due to list overflow.\n"); + pure_names_valid = qfalse; + } + + // Check for pure list names systeminfo overflow + if (pure_names_valid && systeminfo_base_length + download_base_length + pure_strings.hash.length + + pure_strings.name.length + SYSTEMINFO_RESERVED_SIZE >= + BIG_INFO_STRING) + { + Com_Printf("NOTE: Not writing optional sv_pakNames value due to systeminfo overflow.\n"); + pure_names_valid = qfalse; + } + } + + if (download_valid && pure_valid) + { + // Check for download entries not on pure list + for (i = 0; i < download_list.entry_count; ++i) + { + reference_list_entry_t *entry = &download_list.entries[i]; + if (!hash_in_reference_list(&pure_list, entry->hash)) + { + Com_Printf( + "WARNING: Skipping download list pak '%s/%s' due to being missing" + " from pure list. Check to make sure all download manifest contents are also" + " included in the pure manifest.\n", + entry->mod_dir, entry->name); + entry->disabled = qtrue; + } + } + + // Rebuild download strings to account for potential disabled entries from the previous check + refstrings_free(&download_strings); + download_strings = refstrings_generate(&download_list, MAX_DOWNLOAD_LIST_STRING); + } + + // Write output cvars + if (download_valid) + { + Cvar_Set(reference_profile.download_hashes_cvar.c_str(), download_strings.hash.string); + Cvar_Set(reference_profile.download_names_cvar.c_str(), download_strings.name.string); + } + if (pure_valid) + { + Cvar_Set(reference_profile.pure_hashes_cvar.c_str(), pure_strings.hash.string); + } + if (pure_names_valid) + { + Cvar_Set(reference_profile.pure_names_cvar.c_str(), pure_strings.name.string); + } + + // Update download map + if (download_valid) + current_download_maps[client_profile].load(&download_list); + else + current_download_maps[client_profile].reset(); + + // Free temporary structures + reflist_free(&download_list); + reflist_free(&pure_list); + refstrings_free(&download_strings); + refstrings_free(&pure_strings); +} + +static void fs_clear_reference_cvar(const char *cvar_name) +{ + // Make sure the systeminfo flag isn't set - it isn't necessary since these values + // are inserted into the the systeminfo directly in SV_SetConfigstring + cvar_t *cvar = Cvar_Get(cvar_name, "", CVAR_ROM); + Cvar_Set(cvar_name, ""); + cvar->flags &= ~(CVAR_SERVERINFO | CVAR_SYSTEMINFO | CVAR_ALTERNATE_SYSTEMINFO); +} + +void fs_generate_reference_lists(void) +{ + // Generate download and pure lists for server and set appropriate cvars + + // Clear existing reference cvars + for (int i = 0; i < CLIENT_PROFILE_COUNT; ++i) + { + clientProfile_t client_profile = (clientProfile_t)i; + FS_ReferenceProfile reference_profile(client_profile); + fs_clear_reference_cvar(reference_profile.download_hashes_cvar.c_str()); + fs_clear_reference_cvar(reference_profile.download_names_cvar.c_str()); + fs_clear_reference_cvar(reference_profile.pure_hashes_cvar.c_str()); + fs_clear_reference_cvar(reference_profile.pure_names_cvar.c_str()); + } + + // Generate reference lists + for (int i = 0; i < CLIENT_PROFILE_COUNT; ++i) + { + clientProfile_t client_profile = (clientProfile_t)i; + fs_generate_reference_lists_profile(client_profile); + } +} + +/* ******************************************************************************** */ +// Download map functions +/* ******************************************************************************** */ + +void fs_update_client_download_map(clientProfile_t client_profile, FS_DownloadMapShared *&download_map) +{ + FSC_ASSERT(client_profile >= 0 && client_profile < CLIENT_PROFILE_COUNT); + if (download_map) + delete download_map; + download_map = new FS_DownloadMapShared(current_download_maps[client_profile]); +} + +void fs_free_client_download_map(FS_DownloadMapShared *&download_map) +{ + if (download_map) + delete download_map; + download_map = 0; +} + +fileHandle_t fs_open_download_pak(FS_DownloadMapShared *download_map, const char *path, unsigned int *size_out) +{ + // Opens a pak on the server for a client UDP download + if (download_map) + return download_map->open_download_pak(path, size_out); + return 0; +} + +#endif // NEW_FILESYSTEM diff --git a/src/filesystem/fs_tremulous.cpp b/src/filesystem/fs_tremulous.cpp new file mode 100644 index 000000000..52aff4113 --- /dev/null +++ b/src/filesystem/fs_tremulous.cpp @@ -0,0 +1,260 @@ +/* + Copyright (C) 2016 Victor Roemer (wtfbbqhax), . + Copyright (C) 2000-2013 Darklegion Development + Copyright (C) 1999-2005 Id Software, Inc. + Copyright (C) 2015-2019 GrangerHub + + This file is part of Tremulous. + + Tremulous is free software; you can redistribute it + and/or modify it under the terms of the GNU General Public License as + published by the Free Software Foundation; either version 3 of the License, + or (at your option) any later version. + + Tremulous is distributed in the hope that it will be + useful, but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with Tremulous; if not, see +*/ + +#ifdef NEW_FILESYSTEM +#include +#include +#include "fslocal.h" + +// This file is used for some Tremulous-specific filesystem functions + +/* ******************************************************************************** */ +// Core Resource Precedence +/* ******************************************************************************** */ + +// This section is used to determine the priority of core resources such as pk3s that +// are included in game releases. + +struct core_resource_s { + unsigned int hash; + const char *special_id; + + core_resource_s(int _hash, const char *_special_id = 0) + { + hash = (unsigned int)_hash; + special_id = _special_id; + } +}; + +#define CORE_RESOURCES_1_3 \ + {0, "core_game_dll"}, /* Core dll defined by FS_CORE_GAME_DLL_NAME */ \ + -15297494, /* trem13/vms-gpp-v1.3.0-alpha.0.14.11.pk3 */ \ + -1117796085 /* trem13/data-v1.3.0-alpha.0.14.11.pk3 */ \ + +#define CORE_RESOURCES_GPP \ + -1154612609, /* gpp/vms-gpp1.pk3 */ \ + -1688908842 /* gpp/data-gpp1.pk3 */ \ + +#define CORE_RESOURCES_1_1 \ + -1286840555, /* base/vms-1.1.0.pk3 */ \ + 1428306337, /* base/data-1.1.0.pk3 */ \ + -537036296, /* base/map-uncreation-1.1.0.pk3 */ \ + -1768007739, /* base/map-tremor-1.1.0.pk3 */ \ + -2044057604, /* base/map-transit-1.1.0.pk3 */ \ + 2047591756, /* base/map-niveus-1.1.0.pk3 */ \ + -2034645069, /* base/map-nexus6-1.1.0.pk3 */ \ + 1167370113, /* base/map-karith-1.1.0.pk3 */ \ + -2177599, /* base/map-atcs-1.1.0.pk3 */ \ + -1985258489 /* base/map-arachnid2-1.1.0.pk3 */ \ + +typedef std::vector core_resource_set_t; +static auto core_set_default = core_resource_set_t({CORE_RESOURCES_1_3, CORE_RESOURCES_GPP, CORE_RESOURCES_1_1}); +static auto core_set_1_1 = core_resource_set_t({CORE_RESOURCES_1_1, CORE_RESOURCES_GPP, CORE_RESOURCES_1_3}); +static auto core_set_gpp = core_resource_set_t({CORE_RESOURCES_GPP, CORE_RESOURCES_1_1, CORE_RESOURCES_1_3}); + +static core_resource_set_t &core_set_current() +{ + switch (fs_current_profile()) + { + case FS_PROFILE_1_1: + return core_set_1_1; + case FS_PROFILE_GPP: + return core_set_gpp; + default: + return core_set_default; + } +} + +int core_pk3_position(unsigned int hash) +{ + // Determines the precedence value of a core pk3 (higher value equals higher precedence) + if (!hash) + return 0; + auto core_set = core_set_current(); + size_t length = core_set.size(); + for (int i = 0; i < length; ++i) + { + if (core_set[i].hash == hash) + return length - i; + } + return 0; +} + +int core_special_position(const char *special_id) +{ + // Determines the precedence value of a resource indicated by a special identifier string + // This allows special content used by the engine such as game dlls to be included in the + // resource lists and get a precedence value relative to the core pk3s + auto core_set = core_set_current(); + size_t length = core_set.size(); + for (int i = 0; i < length; ++i) + { + if (core_set[i].special_id && !Q_stricmp(special_id, core_set[i].special_id)) + return length - i; + } + return 0; +} + +/* ******************************************************************************** */ +// Base Directory & Mod Precedence +/* ******************************************************************************** */ + +FS_ModType fs_get_mod_type(const char *mod_dir, bool prioritize_fs_basegame) +{ + if (*current_mod_dir && !Q_stricmp(mod_dir, current_mod_dir)) + return MODTYPE_CURRENT_MOD; + if (!Q_stricmp(mod_dir, BASEGAME_OVERRIDE)) + return MODTYPE_OVERRIDE_DIRECTORY; + + if (!Q_stricmp(mod_dir, fs_basegame->string)) + { + if (prioritize_fs_basegame) + return MODTYPE_FS_BASEGAME; + + // Always prioritize fs_basegame if it is different from any of the regular base directories + if (Q_stricmp(fs_basegame->string, BASEGAME_1_3) && Q_stricmp(fs_basegame->string, BASEGAME_GPP) && + Q_stricmp(fs_basegame->string, BASEGAME_1_1)) + return MODTYPE_FS_BASEGAME; + } + + switch (fs_current_profile()) + { + case FS_PROFILE_1_1: + if (!Q_stricmp(mod_dir, BASEGAME_1_1)) + return MODTYPE_BASE3; + if (!Q_stricmp(mod_dir, BASEGAME_GPP)) + return MODTYPE_BASE2; + if (!Q_stricmp(mod_dir, BASEGAME_1_3)) + return MODTYPE_BASE1; + break; + + case FS_PROFILE_GPP: + if (!Q_stricmp(mod_dir, BASEGAME_GPP)) + return MODTYPE_BASE3; + if (!Q_stricmp(mod_dir, BASEGAME_1_1)) + return MODTYPE_BASE2; + if (!Q_stricmp(mod_dir, BASEGAME_1_3)) + return MODTYPE_BASE1; + break; + + default: + if (!Q_stricmp(mod_dir, BASEGAME_1_3)) + return MODTYPE_BASE3; + if (!Q_stricmp(mod_dir, BASEGAME_GPP)) + return MODTYPE_BASE2; + if (!Q_stricmp(mod_dir, BASEGAME_1_1)) + return MODTYPE_BASE1; + break; + } + + return MODTYPE_INACTIVE; +} + +/* ******************************************************************************** */ +// System file manager functions +/* ******************************************************************************** */ + +/* +============ +FS_OpenWithDefault + +Wrapper for Sys_OpenWithDefault() +============ +*/ +static bool FS_OpenWithDefault( const char *path ) +{ + if( Sys_OpenWithDefault( path ) ) + { + // minimize the client's window + Cmd_ExecuteString( "minimize" ); + return true; + } + + return false; +} + +/* +============ +FS_BrowseHomepath + +Opens the highest priority source directory (sourcedir 0) in default file manager +This directory will be the write directory (unless in read-only mode) and will contain any + updated config files, log files, etc. It may or may not actually be the homepath depending + on the configuration of the fs_dirs cvar. +============ +*/ +bool FS_BrowseHomepath( void ) +{ + char path[FS_MAX_PATH]; + if(fs_generate_path_sourcedir(0, 0, 0, 0, 0, path, sizeof(path))) + { + if( FS_OpenWithDefault( path ) ) + return true; + } + + Com_Printf( S_COLOR_RED "FS_BrowseHomepath: failed to open the homepath with the default file manager.\n" S_COLOR_WHITE ); + return false; +} + +/* +============ +FS_OpenBaseGamePath + +Attempts to open path of form [sourcedir 0]/[fs_basegame]/[baseGamePath] in default file manager +============ +*/ +bool FS_OpenBaseGamePath( const char *baseGamePath ) +{ + char path[FS_MAX_PATH]; + if(fs_generate_path_sourcedir(0, fs_basegame->string, baseGamePath, FS_CREATE_DIRECTORIES, + FS_ALLOW_DIRECTORIES|FS_CREATE_DIRECTORIES, path, sizeof(path))) + { + if( FS_OpenWithDefault( path ) ) + return true; + } + + Com_Printf( S_COLOR_RED "FS_OpenBaseGamePath: failed to open game path with the default file manager.\n" S_COLOR_WHITE ); + return false; +} + +/* +============ +FS_OpenModPath + +Attempts to open path of form [sourcedir 0]/[modPath] in default file manager +============ +*/ +bool FS_OpenModPath( const char *modPath ) +{ + char path[FS_MAX_PATH]; + if(fs_generate_path_sourcedir(0, modPath, 0, FS_CREATE_DIRECTORIES, + FS_ALLOW_DIRECTORIES|FS_CREATE_DIRECTORIES, path, sizeof(path))) + { + if( FS_OpenWithDefault( path ) ) + return true; + } + + Com_Printf( S_COLOR_RED "FS_OpenModPath: failed to open mod path %s with the default file manager.\n" S_COLOR_WHITE, modPath ); + return false; +} + +#endif // NEW_FILESYSTEM diff --git a/src/filesystem/fs_trusted_vms.cpp b/src/filesystem/fs_trusted_vms.cpp new file mode 100644 index 000000000..33b4c8236 --- /dev/null +++ b/src/filesystem/fs_trusted_vms.cpp @@ -0,0 +1,415 @@ +/* +=========================================================================== +Copyright (C) 2017 Noah Metzger (chomenor@gmail.com) + +This file is part of Quake III Arena source code. + +Quake III Arena source code is free software; you can redistribute it +and/or modify it under the terms of the GNU General Public License as +published by the Free Software Foundation; either version 2 of the License, +or (at your option) any later version. + +Quake III Arena source code is distributed in the hope that it will be +useful, but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with Quake III Arena source code; if not, write to the Free Software +Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA +=========================================================================== +*/ + +#ifdef NEW_FILESYSTEM +#include "fslocal.h" + +// This section is used to check the hash of game QVMs against a list of known trusted +// mod hashes, to allow QVMs that would otherwise be blocked by download folder restrictions. +// This is strictly a security safeguard against malicious VMs; it is not intended to +// prioritize or determine the "correct" VM for a given mod. A VM being in the trusted list +// does not necessarily mean it is even compatible with ioquake3, only that it belongs to a +// "legitimate" mod and is likely not malicious. + +#define SHA(b0, b1, b2, b3, b4, b5, b6, b7) \ + { \ + 0x##b0, 0x##b1, 0x##b2, 0x##b3, 0x##b4, 0x##b5, 0x##b6, 0x##b7 \ + } + +static const unsigned int trusted_vms[][8] = { + // Original Quake 3 VMs + SHA(4ea18569, bf56a282, d26dc89e, b9efcc5e, edbe0b69, c10182fc, 38446174, c1e55b49), // baseq3/pak8.pk3 cgame + SHA(3a6fd12b, 889f5d35, df20a09b, 51bf8eca, 46966d01, 4be55ffa, d38ddc2f, fb38c807), // baseq3/pak8.pk3 ui + SHA(57c52bf2, 2e4f528c, 064f8af1, 553a7103, 723bab0a, 02276bb1, 1eed944b, f829b219), // baseq3/pak8.pk3 qagame + SHA(c1935b68, bda4a225, dfe50893, a361c00e, 660c9d70, 461f2fa2, f6f89b4b, 4535332c), // baseq3/pak7.pk3 cgame + SHA(73dde0b0, 383b745e, 39806769, 650a18d7, 79c8e8cd, 1ceb5984, 1f6e88b2, fc23861d), // baseq3/pak7.pk3 ui + SHA(407b8ec3, 8e6347cf, 186d31a5, 1629a9cd, 8da5d5c1, 9bc4b5d1, a489672e, 2a275666), // baseq3/pak7.pk3 qagame + SHA(d39dd311, d590550c, 53fb328e, c384ae6f, 09f4a67b, 655209fa, e50cd919, 3c046c80), // baseq3/pak6.pk3 cgame + SHA(bab2fd92, f58d0b5b, d1dc7f29, 6417fba1, 89e10aa3, 76c91424, b048ae0c, 55f29c6e), // baseq3/pak6.pk3 ui + SHA(961a2e2f, a713c139, f32de270, dde81420, cd33aa63, 9d0eb79f, 41ec3b72, ee52e82b), // baseq3/pak6.pk3 qagame + SHA(c769f4f4, ea301442, a4accdf0, 4609e3ff, 60dee762, db09c663, 28232645, 73d38bba), // baseq3/pak4.pk3 cgame + SHA(2bfb85cc, be44b8fa, 05750eee, 85254fc8, f24afcec, c7ca5fd2, 6126305d, a4d696e9), // baseq3/pak4.pk3 ui + SHA(805ee625, 5382a782, 1c438aaa, 5a330ca8, 8184ebcd, 44277629, e6308d13, a5e473e5), // baseq3/pak4.pk3 qagame + SHA(6a9d927d, a75d628a, 5483bb52, a6add9a3, dfd50224, 7a73a086, ec979db6, 97420193), // baseq3/pak3.pk3 cgame + SHA(88d64b9d, 152b4a63, 246fe731, 16e565eb, ff178ca8, b414ee62, b2cb4d18, 88b94762), // baseq3/pak3.pk3 ui + SHA(76e79b92, e6e42552, c2dcf7b4, 282798b1, f510cc22, 4bfb5393, f79ef4db, 45ae922a), // baseq3/pak3.pk3 qagame + SHA(10976bbf, b03ece66, deab2b03, ce7905e1, 7b41da1d, 61d2d070, e386be39, 47c71106), // baseq3/pak2.pk3 cgame + SHA(88d64b9d, 152b4a63, 246fe731, 16e565eb, ff178ca8, b414ee62, b2cb4d18, 88b94762), // baseq3/pak2.pk3 ui + SHA(afc82d75, 0ab836a8, 9233888a, badf7663, f9516093, a679ba0d, ecde6f99, 6aafec1d), // baseq3/pak2.pk3 qagame + SHA(bb77ad2b, a5373f40, d2ed1a0d, 78ec5b94, fa74a002, 5cb61dbe, 2b18d255, 5b15f4c9), // baseq3/pak1.pk3 cgame + SHA(f24823cc, 938eb636, 1a24ab8f, d10c6d20, ee208cbd, e5927c80, 16c47832, 859aa2b8), // baseq3/pak1.pk3 ui + SHA(b477f6e5, a2bcd8d7, 92875177, 49ca6931, b1e87464, c50c1737, c0d840fa, 5168fc98), // baseq3/pak1.pk3 qagame + SHA(ee31bdb9, 865c3e11, afdff3b5, f65dbe95, 99de9527, f2493a25, a347a8b0, ce5eb098), // baseq3/pak0.pk3 cgame + SHA(826a342a, 108ac8a7, fa45f4e7, 52dfa5be, 50fa5ddf, 4fdb87d7, c404d833, c4989627), // baseq3/pak0.pk3 ui + SHA(73d07e34, 1bd21bff, 3e7ec2c9, 61ea9caf, e7ff9150, e0ddcc5d, 8055757e, dece0f72), // baseq3/pak0.pk3 qagame + SHA(6ac3a861, aa28e21c, 2710bc89, 3fc2f30c, ae8e4218, 35e239f1, 35c89f1f, 514f53bb), // missionpack/pak3.pk3 cgame + SHA(ef956cca, 46edbb5c, fc38d091, 27c2788b, c3d5842d, f6de07b3, fa9e553a, 98ba4e5b), // missionpack/pak3.pk3 ui + SHA(48435ea5, 770eaee8, 457c1fa7, 057b6efd, fd919aa7, e66b05bb, 06575744, 67d4f183), // missionpack/pak3.pk3 qagame + SHA(1a650d91, 65c44a97, 87725257, b397ef58, 23af4e5c, 28bcfbd7, 6b647495, 773a9fba), // missionpack/pak2.pk3 cgame + SHA(1f071a96, 2540cf34, b17feb12, 5299ed9b, 77abb483, 12248f17, 1dcab34a, ea137155), // missionpack/pak2.pk3 ui + SHA(f440e701, 4b3987c0, 8cafa843, 5533f4b7, 50c7274d, bbc836f9, 96a437a1, 7a7dfff7), // missionpack/pak2.pk3 qagame + SHA(197732a5, ea8a09fb, b67af801, 6c0b4116, 8196f627, 5e2d6356, 81371750, eb4134e2), // missionpack/pak1.pk3 cgame + SHA(37d55455, 7d45b2f5, d5fdfa9e, 6f362f45, f4040fe0, d828005c, f577f3f4, abe8410e), // missionpack/pak1.pk3 ui + SHA(0fe0f3e3, 0104a272, a6d1771e, a69120dd, d225f238, d3621554, 1b3e1587, 8a2b66e6), // missionpack/pak1.pk3 qagame + SHA(09d0b6eb, 41ea623d, 67031d2d, 7a73058c, cb3bc655, 6ec044ea, d529d48b, 58d15f4c), // missionpack/pak0.pk3 cgame + SHA(7b157f32, acdb21a3, 904d0782, 96672ed2, d32195c5, b7a20692, 2f6f7d33, c6c40e40), // missionpack/pak0.pk3 ui + SHA(da041f17, f296feea, f8269eab, c9062cef, decddfd2, 4ff4d84e, b291902e, 527d1d8a), // missionpack/pak0.pk3 qagame + + // Defrag + SHA(72d9fcff, db4e6650, f50c284a, e8ca9626, c47f6e95, a81c201a, 41f599e5, + 4e82d599), // 1.91.08 zz-defrag_vm_191.pk3 cgame + SHA(4d5cf74a, 5e25367d, 6d14edfb, 06b8b62b, 4da4a08f, 7507d898, b7c8f830, + a3e85aca), // 1.91.08 zz-defrag_vm_191.pk3 ui + SHA(5e5de89f, 68c8e5c9, 4bd6a9d6, a1536c6e, 1cf4b1b3, a9a3feac, 701a0067, + dab1c3a6), // 1.91.08 zz-defrag_vm_191.pk3 qagame + SHA(e0241055, 9cad5dfd, bbfe07e8, 2d285df1, 7ccbbebb, 7c713c57, 2053c01f, + 524d2ad6), // 1.91.09 zz-defrag_vm_191.pk3 cgame + SHA(6c66c6fe, 83394dc7, caf3c208, 1ad2fecc, 53882ac4, 4e1efa7b, da88e164, + ac28b05f), // 1.91.09 zz-defrag_vm_191.pk3 ui + SHA(cbac543e, c646241a, 94b725d1, 6cee4d6c, b2ed7c21, 44097365, cdfce365, + f3ecbd18), // 1.91.09 zz-defrag_vm_191.pk3 qagame + SHA(4de95184, aa5e4a6e, 6e0c67a8, 16719a59, c5be4192, 14977391, cc5da990, + 2e35f030), // 1.91.12 zz-defrag_vm_191.pk3 cgame + SHA(5312ecd0, b18c704b, 4d37001a, 63387d08, 6f8dfabf, 0af91a7f, fab1eed3, + 71e6d857), // 1.91.12 zz-defrag_vm_191.pk3 ui + SHA(76c97520, 32c4b7a1, 427c0e75, a7b9427a, 18cc88d7, 28fd6109, f9d03c73, + eb88fc62), // 1.91.12 zz-defrag_vm_191.pk3 qagame + SHA(dad41168, cd655d7c, b04d4ee4, bf55166e, 0447a92c, f0d3d068, 8cd903d1, + 14645f98), // 1.91.13 zz-defrag_vm_191.pk3 cgame + SHA(f19e47e9, 728ff36d, 6c0558f0, 2d742102, 37675ab4, 3f21ea1d, 6341dcc6, + 3b9cdcbe), // 1.91.13 zz-defrag_vm_191.pk3 ui + SHA(8563ec68, 5588591c, 6f06126e, bd3345e7, b70ce9ed, 8b8e809d, c712e4db, + 915b3c57), // 1.91.13 zz-defrag_vm_191.pk3 qagame + SHA(ef0558ed, 5ba81cbc, 18234a58, 6b3adc7e, b4b8609f, 02127637, 234c5655, + 19bd00f4), // 1.91.14 zz-defrag_vm_191.pk3 cgame + SHA(e057a5ff, 44fd6ff5, 5012c35a, 3106c2fc, ee68e0a0, 4d660839, 9b36ec25, + 344fb802), // 1.91.14 zz-defrag_vm_191.pk3 ui + SHA(61f04e71, 3b4cdc94, 5ea6e532, 54a65d1e, a21a40c7, e9825379, 39509b68, + 5128bc62), // 1.91.14 zz-defrag_vm_191.pk3 qagame + SHA(9a4f85b5, 3baa0e96, 57c3c0b3, a1af8eb2, d2f96954, 1b7a5d03, e9117e74, + 82989e1b), // 1.91.15 zz-defrag_vm_191.pk3 cgame + SHA(d758b933, f8454fe9, 52ff65fa, c6dddf6f, 4a9d5151, c3e107ca, 33253f00, + 14770c06), // 1.91.15 zz-defrag_vm_191.pk3 ui + SHA(b86f1d8d, c93c4330, 581ec14a, 6867bf9b, c287ab8a, 6ca139ec, ae78900a, + ae022dfa), // 1.91.15 zz-defrag_vm_191.pk3 qagame + SHA(4a446c8c, 93fd97c6, 85940fd0, 813f4745, a32b3f96, dc7c47ac, f627a2de, + 490b224a), // 1.91.16 zz-defrag_vm_191.pk3 cgame + SHA(ef636acb, a99eede9, 4f420ef3, 481702d1, 65ad6b25, c9eb4c0f, b0ba5867, + 7e0beb53), // 1.91.16 zz-defrag_vm_191.pk3 ui + SHA(32054981, 8ace1ea9, 428097f1, d8c13a9a, e5261853, e9edfb1f, fc306e6b, + 7123677a), // 1.91.16 zz-defrag_vm_191.pk3 qagame + SHA(14a5d251, 5c018ca7, 0fdc76a8, cc1d858f, 3720807f, 798e7990, b5524f74, + 582ea81b), // 1.91.17 zz-defrag_vm_191.pk3 cgame + SHA(58766e5d, 9e592f51, 9ba09d85, cc3f1083, a03f0a64, fc3b1d88, 95826420, + 594634ac), // 1.91.17 zz-defrag_vm_191.pk3 ui + SHA(ce8a1695, 7df90e1d, 25b4afa0, 2e91a61e, 2b7f3e07, f7a4d1c5, a0c461b1, + 2c4d3cc2), // 1.91.17 zz-defrag_vm_191.pk3 qagame + SHA(01b34f3b, 94a9d62b, 6bb17cfa, 40ab1216, 2370f30a, 02afd238, 9b2e7974, + 3f4fe2af), // 1.91.18 zz-defrag_vm_191.pk3 cgame + SHA(f3f5495d, c00bca5e, 6e3b601a, 77bf73a1, 04a367b2, deafa199, 7fbcd8ff, + a7ed1edd), // 1.91.18 zz-defrag_vm_191.pk3 ui + SHA(be70f4d7, eb488016, eb777b2c, 42c1bb77, 34bee663, d08465d6, e4443115, + 8e5e198c), // 1.91.18 zz-defrag_vm_191.pk3 qagame + SHA(70ec42d0, 50364cfd, ee5ebd50, 5b8af875, 499a60c7, 9a251935, f68490c7, + 2d787640), // 1.91.19 zz-defrag_vm_191.pk3 cgame + SHA(d646b072, a6717648, 6c254bbe, cb5e2a6a, e1df3ec8, 7f83fb84, 4f5cc359, + f9a35550), // 1.91.19 zz-defrag_vm_191.pk3 ui + SHA(a81b8fab, 2b3057e4, cc6ae201, 9bc2ccca, c8e83902, ccdb5f7d, d2ca2952, + a3a6eed8), // 1.91.19 zz-defrag_vm_191.pk3 qagame + SHA(e7551948, ac4c5fc5, d50640a6, 0a336b58, ec1fc6b1, cc572e6c, 337f0ec3, + 9fb5b35b), // 1.91.20 zz-defrag_vm_191.pk3 cgame + SHA(72e9f3fb, af044bcf, 88f4e897, 57512f76, cc74abad, 398fa12f, a978cd82, + 52c83077), // 1.91.20 zz-defrag_vm_191.pk3 ui + SHA(553bc0f2, 9c2e4d05, 14ec720a, 1bb0a276, 95b79d72, 0eaf3305, 3091912b, + 70ead9ef), // 1.91.20 zz-defrag_vm_191.pk3 qagame + SHA(33c33558, c63a0a2c, 06bf813e, 9f37f7b8, 7fbf6e38, d517c0f4, bab119ab, + fa2cec03), // 1.91.21 zz-defrag_vm_191.pk3 cgame + SHA(efb2d97c, 4523f370, c50ad8ce, 65b416cc, 804de272, fc7b818f, a3cf4d08, + b95991c8), // 1.91.21 zz-defrag_vm_191.pk3 ui + SHA(04877cfc, 1e769450, e39d841d, bfdf61a4, 8648b7a6, a2d2e928, 179b11d1, + 3dec500b), // 1.91.21 zz-defrag_vm_191.pk3 qagame + SHA(224222a1, b435210a, ea2a980f, 510e626a, bd9e0956, 749840b3, 0fb7e039, + 726b9aca), // 1.91.22 zz-defrag_vm_191.pk3 cgame + SHA(0bc890e4, 5eaa190e, 6c1a1735, 3da7794d, 5a3ba898, 90f99397, e7a947ac, + ce80e75c), // 1.91.22 zz-defrag_vm_191.pk3 ui + SHA(fd83b123, a5d5a243, 0007d0c6, 074070db, d8c03aa1, bfde3684, 365ea60e, + d86f6c3e), // 1.91.22 zz-defrag_vm_191.pk3 qagame + SHA(753a4747, c0e7e2b7, f0172ee2, 2bca1f84, 56f90e81, 5940a36a, f79ba4be, + 44bb57ab), // 1.91.23 zz-defrag_vm_191.pk3 cgame + SHA(52a45d3e, c13533fe, 7cd458a0, 1a787b03, ec6e111a, 61fbbe94, 9328b5e8, + f1e39159), // 1.91.23 zz-defrag_vm_191.pk3 ui + SHA(114edd74, 03235177, 8eff4f2d, 6dce9494, 34f693ec, feda608d, 3d7a53c6, + dc9b97e1), // 1.91.23 zz-defrag_vm_191.pk3 qagame + SHA(f7f814f4, f5960606, 85f3c4f2, 74d994e9, 85014b3f, 91b699ac, bc948089, + 34b990af), // 1.91.24 zz-defrag_vm_191.pk3 cgame + SHA(4b38aac9, ab6b7626, 692cead2, 810ad1a7, 9aea7525, 257d154d, 7e786a79, + b777a46a), // 1.91.24 zz-defrag_vm_191.pk3 ui + SHA(3aefd360, e9dce2bd, f4bddc95, 7e08d1c6, d20cc059, f0500bf3, 71a6eb42, + 01b32f88), // 1.91.24 zz-defrag_vm_191.pk3 qagame + SHA(b950f4d7, 895cc4ff, f12877ec, 7f120746, 0b5db3ad, 6c0ecf1c, 44f732b8, + c734344e), // 1.92.00 zz-defrag_vm_192.pk3 cgame + SHA(646e8473, 354906fd, 6a874c90, eaa198c6, 127d5193, 89702b63, 71e35321, + 32010b4e), // 1.92.00 zz-defrag_vm_192.pk3 ui + SHA(e8fc046c, cf168b9a, 15c45e8c, 600913f3, e6b30b36, 691ea45a, b087c6ce, + 5af8a846), // 1.92.00 zz-defrag_vm_192.pk3 qagame + SHA(5ebedeb4, 840ce409, 0a2ebce2, 0836514c, 74bd6913, 890bb877, 3c567ad7, + f8c6b868), // 1.92.01 zz-defrag_vm_192.pk3 cgame + SHA(e3b23641, 46899012, 12cc5da1, 2416a1b1, 40267dd3, b421247b, 3bdc4e97, + ff61b6e6), // 1.92.01 zz-defrag_vm_192.pk3 ui + SHA(8191934f, 8f9f49de, 0bb4942b, 70acf9fc, 22b4a921, c72fc0c3, 3e85cdcb, + 243c18de), // 1.92.01 zz-defrag_vm_192.pk3 qagame + SHA(a785d303, 4443f3fb, 47ff81c9, dcf7520f, cb025bcb, 09d947f4, 52fdd9e6, + 263adb3f), // 1.92.02 zz-defrag_vm_192.pk3 cgame + SHA(f7c7edea, 6e95a626, 25a6f958, 8c559edb, 07030c88, 9984bdde, 47d77d62, + 819a75ab), // 1.92.02 zz-defrag_vm_192.pk3 ui + SHA(e6af8c30, 17e1fa32, cdc38444, 7491017b, 7b12d5dc, 73c0662e, c69a294d, + 5748e117), // 1.92.02 zz-defrag_vm_192.pk3 qagame + + // OSP Tourney + SHA(48e17665, b0b274ba, a9567036, 4364be37, 3fbbda4a, ea6e74c1, fb587aaa, 5feb33b3), // zz-osp-pak3.pk3 cgame + SHA(999e49d1, c24cdaf0, c4eff5ac, 1241549a, b7559968, 0eea3a73, e0ed07bc, 1600f8be), // zz-osp-pak2.pk3 cgame + SHA(82b0c71b, 3123e535, c3be3e31, 511ae554, c720d6c7, d8ce7bb4, 5a54ef5a, 8ef5c710), // zz-osp-pak1.pk3 cgame + SHA(36133dea, 02f56992, a8683f85, 2893a414, e330cc87, 06389567, 1ee268b5, 1891147c), // zz-osp-pak1.pk3 ui + SHA(b6ccc3fb, 79390a4c, 5222c0bd, 687eef06, a086632c, 23faf1a2, ca34a3a6, 4157fb32), // zz-osp-pak0.pk3 cgame + SHA(e693f49f, b21805e6, c7d232ae, 3a6994dc, f7f8adf3, 19dfb0b5, 7af6bfc1, ed3075a9), // zz-osp-pak0.pk3 ui + SHA(7ce6a6ac, 1fcb4442, 06461e19, 129d4f5c, d1c4ba79, a3cbf881, 653e6450, b8277160), // zz-osp-server3a.pk3 qagame + + // Challenge ProMode Arena + SHA(7bb9a938, 173d804e, 3c9a7d3c, 5a923f32, a1620095, cb953200, c93142e9, 7d1e5a87), // z-cpma-pak148.pk3 cgame + SHA(d9febc97, d8eb1732, efaa8a35, b6d344de, 5dcf6e4e, 94ce791a, c3b16aa9, 1884142b), // z-cpma-pak148.pk3 ui + SHA(b7998090, e8864e43, 66e8daac, 14c3820e, 96d7d0a0, b6045985, 6452299f, 335c0958), // z-cpma-pak148.pk3 qagame + SHA(95149049, 5ae9bcaf, 77ca8ca9, f09470c5, ec448f54, ee774514, cce8c039, 2c7c8999), // z-cpma-pak146.pk3 cgame + SHA(eba6fc2d, 6f1bf3fe, 384eb6b7, 0c30a72b, 979fcb4a, 3c11ebe2, 6ed2d76e, 1321c6c2), // z-cpma-pak146.pk3 ui + SHA(deeae01e, b23fdaa2, dcef0b50, 5e21abd5, 53316fc6, 294ff6e4, 067aa616, 1d4cedec), // z-cpma-pak146.pk3 qagame + + // Threewave CTF + SHA(14858804, fb98609e, d8b3b3c3, b825f0a7, cb544063, f7e43735, c884cd5e, 4a51157c), // pak05.pk3 cgame + SHA(259fa9a6, 10659b0a, 3b1c59b0, d83100dd, 234cfe0f, a0ae8fb3, a891aca3, e0c722a7), // pak05.pk3 ui + SHA(9751bad9, 9a2d138f, 96a9b043, 6d2ea2d9, 65b86214, 175dc33e, 4cea95e0, 59419337), // pak05.pk3 qagame + SHA(f13c94b3, 60571ab1, a3735eeb, 8a70c1fa, f6049f9d, 0914fb87, 7a7776b5, 020135dc), // pak04.pk3 cgame + SHA(712ee5a9, 8a60021b, 7fc36601, dfed4cfc, 68c51b48, d04970ab, 92b69384, 026d9ad0), // pak04.pk3,pak03.pk3 ui + SHA(1a9a5122, ee120684, 00ea0496, 001b9485, 8161540e, 912207e3, 88721781, d984daee), // pak04.pk3 qagame + SHA(472cb89b, ca6681b6, 8d2a790a, b10affa3, ddfa19a6, c1c798ce, 6a0ea566, 86a51daf), // pak03.pk3 cgame + SHA(3fc47bab, 74f822c1, 5922cb08, e8640cd4, 363bf8df, 2716f141, d2a4ccd8, ab5a8ef5), // pak03.pk3 qagame + SHA(6277405a, ff2288e8, 468fb757, fca680c7, 3be6d43e, 4cceecc3, b4dd2f81, 32372fb7), // pak02.pk3 cgame + SHA(247274bf, 64787787, f89503ba, 9e9c95d7, b113404c, 699bab8d, c5b57aa4, 5c0a4678), // pak02.pk3 ui + SHA(8f5fde11, 841c00cc, 43877f32, ad8081d1, 6fcd5859, 9974e42b, 8334f5fd, 3209fa0a), // pak02.pk3 qagame + SHA(661588fa, c009255a, 257ebb49, a12a279e, 19529b97, a6bb610b, 1e0fa6c2, a23175b1), // pak01.pk3 cgame + SHA(1a655c47, d3506acc, c9ae0332, a3f544a8, a0ff8816, cf4001e7, 723e37d9, d2af97a9), // pak01.pk3 ui + SHA(7e68a480, 79a22404, 2a3b8fb8, d7bfc6ba, becc6dce, d5b75796, 01927477, 8b45c987), // pak01.pk3 qagame + SHA(d462e4dc, a60dd4b9, 38e9f937, 90c8e569, 64a121cb, de8e8b6a, 0022b8cb, d2a4bb7a), // pak00.pk3 cgame + SHA(c8a59247, 7737802e, 5eb5ec20, 7f9afaa7, 84ba8449, b53726d3, d0d59018, bffffc6f), // pak00.pk3 ui + SHA(a587f8ce, 9f34b1ce, 0aa692d0, 0cd7418b, 1834c808, 12f84c8c, 6f56e1e6, 8e2a9686), // pak00.pk3 qagame + + // ExcessivePlus + SHA(b73dd1f4, a69204ab, e532ed2f, 3b697fb4, e86d14bf, 7408a2e8, db4696e4, 3708b45d), // z-xp-2_3.pk3 cgame + SHA(2ded5963, bc992ec4, ca8068ec, 0b396116, b77e46ba, 75fd7694, dccf2686, ef78b2c0), // z-xp-2_3.pk3 ui + SHA(2582a9b3, ececd83d, 541a667b, ef035cec, 7d70ba77, 86c9936a, 6ecf655a, c0ac4dc4), // z-xp-2_3.pk3 qagame + SHA(29187bc7, 17bd375c, ee0bb8fe, f00d02f0, baedef3d, b473619d, b426f22d, 49abfcdb), // z-xp-2_2b.pk3 cgame + SHA(ee7cd2b1, efae49a3, 67155113, 7559a3f2, 30b31be0, e45b147b, e42429f0, 0d32979b), // z-xp-2_2b.pk3 ui + SHA(d9ada8cd, ec1d31ae, 419a1813, 81aca870, 7f224adc, e0720225, b05b210f, d88d38dd), // z-xp-2_2b.pk3 qagame + SHA(16e25f30, 43d2d32e, 43ef1c04, a21e201e, 4ea02f30, 970a8548, d5b46509, e3eceba0), // z-xp-2_2a.pk3 cgame + SHA(19339573, 8667ccb2, cf875910, 93906c8a, 5a2a2dba, 6883f776, 3a4fcc7a, 756b42a8), // z-xp-2_2a.pk3 ui + SHA(8a18446c, 33f53d30, 718641f1, 6ccdd00e, 85b21e26, b9b1bde4, f953cb3a, d0cd71dc), // z-xp-2_2a.pk3 qagame + SHA(dd68b033, 5a68acd5, 994c4602, d1909194, b3b5a03c, 4ebfeec2, 82a54a11, 0ab47a9a), // z-xp-2_1.pk3 cgame + SHA(6f29922c, c8b9f2e3, 8a7ed426, 578e8b85, 8e22d59b, 65aea126, 5e0c64ef, 62bf419b), // z-xp-2_1.pk3 ui + SHA(cd3c90de, 45f01a3d, 1bbbbb90, b75bb050, 92dfb495, 27b74d38, e5cd7c5e, 3611ed81), // z-xp-2_0a.pk3 cgame + SHA(0ed23cb9, f2b78509, 50008405, 6f81358b, ea81f19f, 4e7567db, eb7ceb16, 9ff1bde2), // z-xp-2_0a.pk3 ui + SHA(00b31196, e46717cb, 0fd238ee, 2c48120b, 22ebb95e, 4cbdda57, 19b0b038, 8723e395), // z-xp-2_0a.pk3 qagame + + // Excessive Dawn (edawn-mod.org) + SHA(fcbca7d3, b3dbf15a, b13e9735, 9cf57ed3, 5a8428ee, b7b3f822, 51784ab7, 69b1d74b), // z-edawn-1_6_1.pk3 cgame + SHA(ebd474f4, 1c524cb4, 3d46838e, 4dae7706, 4ee5b2ea, 200796bd, 34f62c66, 064f2749), // z-edawn-1_6_1.pk3 ui + SHA(ea2315f6, e2e2a7ec, 011b9423, 927297fe, 6e5793ad, 82159bd5, f458f99b, f2bdc794), // z-edawn-1_6_1.pk3 qagame + SHA(9bdbb62f, 65c1029e, 1a0b5448, 1f1f17e7, 51e15718, c98146ab, 7e0badbc, c38d5bb4), // z-edawn-1_6_0.pk3 cgame + SHA(9ce6bd3d, ea2e8a51, ec975adb, 66c2467c, 8419c7c9, e477ac4b, 71b14f9f, 02205edf), // z-edawn-1_6_0.pk3 ui + SHA(228dc026, 89ff82f2, 83b4843b, e0b0d7bf, 345b1391, c308542b, 1a9f2aa4, e5514c28), // z-edawn-1_6_0.pk3 qagame + SHA(42dbb85f, efa29712, 29b8f0a0, 7f9e2e1d, f27b25ff, 32f2c35f, 508cddd6, 774cd8fe), // z-edawn-1_5_2.pk3 cgame + SHA(0183be28, bc6d7ab9, 642bfafe, 6d2245ce, 1855db15, 0bf531fa, cace82b1, 6a335ef7), // z-edawn-1_5_2.pk3 ui + SHA(597effa9, 1008f483, 2a71f195, 35183f1d, 956d6796, 675e4d41, be9db980, 1a854521), // z-edawn-1_5_2.pk3 qagame + SHA(9a4ddeed, c2719c28, 1e4cb08b, 25341087, 4ff0beb0, 805b3ea8, 436e5477, 8ab48829), // z-edawn-1_5_1.pk3 cgame + SHA(c4cb7504, 00e6d960, 8beac8bc, 1655c921, 1c31168c, ed8ff62b, df5140f0, 5eb457ae), // z-edawn-1_5_1.pk3 ui + SHA(e9f3495e, 3fff3cb6, e7c020e0, 67e10394, 15d89f47, f964050e, f45bf710, f3aa6dab), // z-edawn-1_5_1.pk3 qagame + SHA(ba4471e7, 4b57d05d, 6ff474d0, 76b7f161, 59a27e40, b0424f65, cfec561c, 1fa1fc6a), // z-edawn-1_5_0.pk3 cgame + SHA(49ca495a, 13b9bc21, c7e6a4e5, 7feca87f, 08253187, 8d306757, e1968536, e8b52caa), // z-edawn-1_5_0.pk3 ui + SHA(f21140a6, 41400e66, efc89c15, 417dbdc1, e39408e7, e52208c4, 6da2d1b5, 207eb846), // z-edawn-1_5_0.pk3 qagame + SHA(8155ba68, c101afd8, 37f6144f, 386534c4, 9c8ef3fe, 45581b43, 9583bbf4, d9cd57a4), // z-edawn-1_4_5.pk3 cgame + SHA(c7f747ad, 0c90921b, 60ccf3fc, 6084ac85, 493bfa8f, eae558f7, 05d873d2, 1cd1ddcb), // z-edawn-1_4_5.pk3 ui + SHA(2bd54dae, 6d1af4bd, 08c834bf, 943dee9a, 3ee4c9b6, 6234b7dd, d0bb73f9, 40e532f3), // z-edawn-1_4_5.pk3 qagame + SHA(b9ef494f, 584a2475, 0bc48381, de973703, 2c256765, 6fc1f570, e67317a5, f107ff2a), // z-edawn-1_4_1.pk3 cgame + SHA(d51dd432, 4a47ab98, 859a2983, d12c7a2d, 754fd89d, de1afae9, 7affd7a4, ec01bbe2), // z-edawn-1_4_1.pk3 ui + SHA(5169ee13, c93fb8ce, b135fc2e, 9a80fd81, e62f1e38, c122f7fe, b9cbb5c7, 290e4176), // z-edawn-1_4_1.pk3 qagame + SHA(bfc6035d, 6ef78bcd, dfa160c2, fdcff061, d55320cc, 9d9a1b06, b61b9f90, ca16eb7f), // z-edawn-1_4_0.pk3 cgame + SHA(0fb1fb2e, 00ea5c42, 3a381f05, 1c94b2be, 0cfcfa96, 4788d5b0, 80eefaef, e59cea50), // z-edawn-1_4_0.pk3 ui + SHA(25145a1b, d6a072f1, d11085e3, bea7edaf, aede8897, fe7a57c0, b6d4758c, fd6811d6), // z-edawn-1_4_0.pk3 qagame + + // Freeze Tag (nbquakers.com/freezetag.htm) + SHA(a9ad3329, 889a53e8, 66d44507, d4419897, bcf029b4, 5bcae16b, 70c018b0, 5624da3f), // freeze0.pk3 cgame + SHA(152f78f5, 6d17824d, d9fed8e6, a9cfb1f5, 0effa09a, 2fbfdd42, c57811ba, 43c9e192), // freeze1.pk3 qagame + SHA(5e7555b5, ff609515, f447939e, 573537d9, b4d49f38, 66f93378, a5e28c44, 267b2479), // freeze_ta0.pk3 cgame + SHA(bc2a35ba, 7220b11b, 596168e1, f13a4a43, b3957dcd, 9c58f77d, e22159c0, 0c1e08fd), // freeze_ta0.pk3 qagame + + // Ultra Freeze Tag (nbquakers.com/ultrafreezetag.htm) + SHA(db3cae1a, 7bd6c12a, d33c9627, 9c6fd222, 79381e67, 909fabcb, b3f890d8, 1ecb4cff), // ufreeze-22cg.pk3 cgame + SHA(45f85fa7, 5289e55e, d86ff221, 8b568434, 74230e93, ec88fd49, 26cc0797, 9be99531), // ufreeze-22ui.pk3 ui + SHA(e2d8e1d5, 10b0668e, 8bef2606, 584cc1c8, 3736aea2, f9abeafa, 63782ae4, 329991f1), // ufreeze-22sv.pk3 qagame + + // Weapons Factory Arena + SHA(2b430a4d, 56de288c, 44ab40f1, a181313c, 5afdd3e7, 271e43af, 599d1264, abcd2dde), // pak35b.pk3 cgame + SHA(835157d6, 2dc5431e, 9efd6c92, d3a808a6, 622d650a, 06e02db1, e48401ba, f92ea264), // pak35b.pk3 qagame + SHA(188335be, 886d500f, c2a03a9e, ebb75340, 10b84e45, e3f36a1d, 92ad861f, 2bea01a3), // pak35.pk3 cgame + SHA(6867860a, 68af55b3, 90fb4ecf, bf3c345f, 3e963ddb, 561097fb, 576df95c, da91a136), // pak35.pk3 ui + SHA(71d80288, 6651181d, 94308311, c3f6fd18, 3df7fd0d, 6a57895e, b7896ecf, 05817eca), // pak35.pk3 qagame + + // Rocket Arena + SHA(1e234470, 41681f10, 05e1ccb5, 62d2eb90, 475512a9, 18525fbe, dc1bbb0b, 570365c7), // vm.pk3 cgame + SHA(5f871837, 4daa2fda, cff14fdb, 2d2b4f71, 4d37727f, 38f8e5c7, b6250419, 77d14c39), // vm.pk3 ui + + // True Combat + SHA(ffddc7fa, 33e2444c, 23996a46, 31920601, 42e76162, 211daa31, 0bf6e63b, 810b3b87), // truecombat/pak2.pk3 cgame + SHA(3764ef13, c8613837, 22088147, bfc66fd8, 05471168, ad6117d7, e56a570d, b91a961b), // truecombat/pak2.pk3 ui + SHA(d3c7c5c6, 34a4df76, c6cac48b, cd4024e4, 9dbb7886, a806f17d, 74b4b583, 5fccbad4), // truecombat/pak2.pk3 qagame + SHA(506fe6bf, 66897cba, da2b8249, 565b9da8, 16eaefe7, 03772f1d, ffcc5e7f, 078cf069), // q3tc045/pak6.pk3 cgame + SHA(46427997, abbd31eb, dc5380f3, 1bceffa8, 3690f10c, 05f46e43, f855a7b4, 70c4ca41), // q3tc045/pak6.pk3 ui + SHA(0fec7933, 50c547ee, 38810f19, 88d6ae6f, fbf3307c, 0cf236bf, 6d0acdac, 1aa3e096), // q3tc045/pak6.pk3 qagame + SHA(c2b325bc, 540d9ecd, 918685e7, 5c864e72, e314848c, 0af1b6d6, 9958f37e, 53fcb07f), // q3tc045/pak5.pk3 cgame + SHA(efce2ab0, 9c5f5bf0, 636a5c74, 5358dad0, fe09cc32, 37e876a6, bffc99d4, 7a2bfe97), // q3tc045/pak5.pk3 ui + SHA(6427e326, 30f715b6, f167837d, 5a7ac908, 7acd4f84, e179eaed, 1003c2c0, 1d89e64b), // q3tc045/pak5.pk3 qagame + SHA(e590ba97, 9191f6ac, 69a00dea, 52219d9f, 255b2dae, 1997fde1, 955a1657, 5d75d4ef), // q3tc045/pak4.pk3 cgame + SHA(f25ee7ef, 50afa697, c8c32a74, fe3cabda, f746714a, e2723247, 8b191b4c, f2116351), // q3tc045/pak4.pk3 ui + SHA(c20dc792, cccb0f8f, f63837a4, aee6f919, 027096c4, af93ec1e, b8429434, a03c4a5e), // q3tc045/pak4.pk3 qagame + SHA(715321a2, 09b9343e, b85b50ba, 6ae64024, a9ed2fa2, ecb925b7, 7ec6461d, 71cf10ff), // q3tc045/pak3.pk3 cgame + SHA(1939108c, e0a4868b, afa904c3, 91f7260c, 3bbcb378, e52958f1, 88ff1895, 421aae19), // q3tc045/pak3.pk3 ui + SHA(41c02d25, 5cb4e95b, f9b46bd3, 72d44a4f, 1d04ad68, e4d4cf22, f9d97a98, 8940728f), // q3tc045/pak3.pk3 qagame + SHA(e2b1692f, 077d6b82, 181fe52e, 104d05e9, 1ae4b070, fae8451c, 3f4ca6d8, 5185bfd5), // q3tc045/pak2.pk3 cgame + SHA(177a43cb, 758ec34b, 7a66c854, 7993f5bb, 74bc7a2a, fcb005b5, fb5c5150, 45c4ce4e), // q3tc045/pak2.pk3 ui + SHA(38eaa136, 9d5365d1, fe38199c, fb23485e, c6630343, 2575d95d, 598c3a5b, 384d1a28), // q3tc045/pak2.pk3 qagame + SHA(af4b9138, 7d9304b5, b425aa1c, 31e9f563, 71ab194f, 086ea9a0, 2b3554ce, be02fe56), // q3tc045/pak1.pk3 cgame + SHA(40053b90, 7c76316d, a3370adf, aba6910b, a9b82c5f, 83e426cb, 5bf36925, 75d143f8), // q3tc045/pak1.pk3 ui + SHA(b9615dd0, e33b52cf, 436b144a, 73e13248, 4b72ef12, 88202771, 3f2cf541, 8d497dfd), // q3tc045/pak1.pk3 qagame + SHA(c1308b35, d0bf4042, 1304bd99, 0a38bda0, c694ffa9, 9d87f746, 328fd6db, 57a636ed), // q3tc045/pak0.pk3 cgame + SHA(954ebd3f, 440f0a08, e1c085bb, ca2a5b4f, 322abb72, 17ed833d, e7b03c33, 727dc028), // q3tc045/pak0.pk3 ui + SHA(59695dfb, daea80d2, d64df7e9, f89a32ec, 4551858b, f0bee5fa, c3b6aaf2, 8b8529c6), // q3tc045/pak0.pk3 qagame + + // Quake Revolution (quake-revolution.net) + SHA(7039ee45, 20113948, ec5a26b1, e22c88ce, 164574c5, ba24240f, 8d062ccb, + 4a096143), // z-Revolution-client-200[Epsilon2].pk3 cgame + SHA(9c12d1da, c8865664, f580c4e1, 5c6195af, b2195197, 4c039319, 05636f07, + b624dcd2), // z-Revolution-server-200[Epsilon2].pk3 qagame + SHA(8c62e928, d0fa7a67, 672bdca8, 32fc4db8, da5800b6, 41270db7, 6bde0326, + e2a5b256), // z-Revolution-client-200[Epsilon1].pk3 cgame + SHA(1f42801b, 6faa1cba, 60c14757, bade8001, 4be2218f, 3b52685e, 6256668f, + a0e36251), // z-Revolution-ui-200[Epsilon1].pk3 ui + SHA(a7da4d5f, 93fc9c07, b018b92d, 91e17dcc, c00f898f, 2654664e, b41ffc6c, + 771cfc7b), // z-Revolution-server-200[Epsilon1].pk3 qagame + SHA(e5d17064, 445a52b7, 5619418c, 2c43251a, 7b589ab3, f6ec3adc, 949697da, + 86680a1d), // z-Revolution-client-200[Delta6].pk3 cgame + SHA(d5c75915, 6311193e, 047e8a95, 4ff24cd4, b59f60f1, a49e1eed, dc4bdfa8, + fb15a793), // z-Revolution-server-200[Delta6].pk3 qagame + SHA(35939d6e, a6aec584, f97939f4, 1f9bb4b4, 8f8dc4d8, 8ea8c07b, 278cd14c, + 12fbb4e1), // z-Revolution-ui-200[Chi1].pk3 ui + SHA(e3ea6e17, 2c73cee8, 1f3601e8, a0fe50ee, aa117fd0, 9dd1948c, 3182e3a2, + ddb600be), // z-Revolution-client-200[Beta1].pk3 cgame + SHA(8032d831, 00c51e60, 7d57d221, a36755fe, c8b374a2, 67059c06, 5beceb93, + 7369621f), // z-Revolution-server-200[Beta1].pk3 qagame + SHA(0d90b458, 0bfd1f09, 7a1d9d11, de223475, 39beacdd, ac0b62d1, 7e8637d5, + 50c0a20b), // z-Revolution-200[Alpha1].pk3 cgame + SHA(02e9bd52, 92119cc0, 75822413, 91a2b7ee, c1c6b2b3, 5f5aa34f, e22e745a, + ca1cf49d), // z-Revolution-200[Alpha1].pk3 ui + SHA(4904e60c, 6d914309, 266c7f97, 97f7009c, e286c128, aa2738c7, dcf7049c, + d72d7952), // z-Revolution-200[Alpha1].pk3 qagame + + // Zero Ping Mode + SHA(d0996f1e, a891741d, a06f6516, c365f7ae, 868cfd7e, 2a02c06f, 24bdf8c2, 2894db10), // v4.3 pak0.pk3 cgame + SHA(a67cb9b2, a241a78e, 3c64d991, 6d0531f1, 5d34721f, 3dcf7bd8, 7e4973e7, 0a197e9c), // v4.3 pak0.pk3 ui + SHA(9f6bb8b3, 02df398b, 1f8283aa, 981d9464, 44f1c150, 36af77fb, 414044b5, b25db659), // v4.3 pak0.pk3 qagame + SHA(06b07548, bb7f528a, f36d4700, 50436a4a, 99ea11d8, 0692d07d, 0f0febb6, e61645fc), // v4.4 (?) pak0.pk3 cgame + SHA(9364bdd2, 2eb684db, 30dd2f54, c5bcfa3f, f069ef31, e827149c, 9a527b2d, 588555ae), // v4.4 (?) pak0.pk3 ui + SHA(f26d2b8c, 7a874cf3, 293e93ec, 9faffd39, a0f94f4a, 02414973, 09e608b6, bd89ebc9), // v4.4 (?) pak0.pk3 qagame + + // RGGMod (rp.servequake.com) + SHA(44c346b6, b0160384, 4b37c192, 4e741cc1, a8ff5260, c6ef1a62, 655c6839, 0254c49f), // RGGmod_1.32b.3.pk3 cgame + SHA(65121467, bd4f0027, 69d02814, 2e2e7efc, 48eb000c, 1784027c, e477e860, 981bf9ca), // RGGmod_1.32b.3.pk3 qagame + + // InstaUnlagged (?) + SHA(25bba8c3, bf14017d, 849e06e7, 5238fae7, 5db08d0e, 4e277ca2, baeed356, 44168ae2), // pak0.pk3 cgame + + // Hunt Mod (lonebullet.com/file/mods/hunt-mod/37111) + SHA(f187545a, 3da187e4, 7a59f1d6, 5cb2bd94, a5335d7f, f0a8f57f, 07e86b78, f6fa8286), // hunt.pk3 cgame + SHA(62230abd, f723d6c0, adbe0f66, bec34f8b, 009a2812, 16bcbc2f, fd2c3e8d, 1494fc6e), // hunt.pk3 ui + SHA(c0fa2c7f, 160c2d82, 6929a560, 7f811d82, e712ede1, 3df6e1fe, e4dc76f1, 02b95431), // hunt.pk3 qagame + + // Western Q3 + SHA(fcb31ce6, 852964b1, 7bad9a86, e64758a5, 4c7daad2, 3b4e471a, 448f827a, e77e62a9), // wq3_pak3.pk3 cgame + SHA(15b75a81, 6615e7ad, e3936783, 22bfba52, c82ae592, 9d44e191, 2368091d, c30f8f9e), // wq3_pak3.pk3 ui + SHA(bf02966b, a41415fa, c9ade96e, c445b1b1, b32d5240, 8fa46e9e, 10f42fe3, 2a28081c), // wq3_pak3.pk3 qagame + SHA(83d5e677, 80d02573, e8fb8c56, 9e202b18, 89637375, bbff0253, e9391c29, e230c88a), // wq3_pak2.pk3 cgame + SHA(f88fd545, 4c7996ae, c58505d6, df3d89a9, cdc62815, 62557314, a72949eb, 96cea3bd), // wq3_pak2.pk3 ui + SHA(2bec1a03, 8e86468e, 611b1dd5, 88e845e0, ba099dba, d98a4e81, 6848fb81, cd046ee5), // wq3_pak2.pk3 qagame + SHA(e3ab1d1d, 99f4c502, 05324a4b, 997a3543, 9d45fdb2, 3901be61, 3684813e, 7aa2ee9e), // wq3_pak1.pk3 qagame + SHA(281765c9, 8c1835fa, d470d970, 87b92e9d, 994c6bcd, b82cf19a, 43cc7945, 3fbba188), // wq3_pak0.pk3 cgame + SHA(703bd9b1, dcf1ecae, 2b553cbf, 0b54c4a9, a7a3da97, ec8574af, a3e4bcf0, 21344c16), // wq3_pak0.pk3 ui + SHA(c6915701, 3293f03f, 46cc5768, 3806bdc8, 75ba0b77, d7263626, 95aa6391, 5c22badb), // wq3_pak0.pk3 qagame + + // World of Padman + SHA(05d765a2, 946f5347, bf36cf35, 22f854b6, fb5097de, 4451c0d8, 66cc57a8, f8f2290d), // wop_006.pk3 cgame + SHA(10e8e292, e5572b29, 2f8388c2, 51950783, 982aa9ed, 676d005c, 7c3e02e3, 3d4591b5), // wop_006.pk3 ui + SHA(e3d80496, beefd222, 8e10c8db, 7aadb312, 592fc134, fb7e21e0, 63766df9, d68e52eb), // wop_006.pk3 qagame + SHA(cfa3973f, 39256a54, abebe253, b65f289f, 675a51a8, dae34907, b68ce305, a5586d57), // wop_001.pk3 cgame + SHA(0ea91c73, d3627e74, 421a1c57, 7c0d9a34, badde572, 7f340322, 2965c0d7, 9edf07e0), // wop_001.pk3 ui + SHA(c089d1b0, b541c6a1, d5e0e444, 099bf968, f0dba16d, 63292801, cbd75ecc, c43944a1), // wop_001.pk3 qagame + + // Reaction Q3 + SHA(ea706460, 01cf5d9e, d4166c37, 62daab83, 099c3647, 64430086, 69a39a42, 26041b12), // reaction1.pk3 cgame + SHA(935c37ac, c683a77f, 4ee1fabd, c832d5f8, 8a95c27a, 525717d2, 08f6cb39, d1b53dcb), // reaction1.pk3 ui + SHA(c55dac68, 36e685ae, 016fd0ed, b4148be0, 9c70e0f9, 92688fb9, 1ed10ff9, 720fc094), // reaction1.pk3 qagame + + // Generations Arena (wireheadstudios.org) + SHA(67b25e2f, f86baf94, bd4874a7, 2677ab0a, 11e8e86e, ab9edc8a, 99bb2d05, 027ef919), // pak9_vm_1.pk3 qagame + SHA(9a4a8035, 8e469578, 90314c5b, eab5c415, 65e5d6eb, a3434515, 20ae380c, e743a85c), // pak9_genv99f.pk3 cgame + SHA(0ad23c52, 6c374935, cf22ce6c, c6cb9710, 79f86a6c, ffd820b2, 9eb06437, 45b60701), // pak9_genv99f.pk3 ui + SHA(cb0a5aaa, 451712f0, 5c334187, c6c8c6f7, 0c9de16e, 76445b6a, fa1c497d, 43b2f390), // pak9_genv99f.pk3 qagame + SHA(694478c9, 6d34e144, 3c3856ef, 0eba207d, 0fe36a3f, bbd5fe0e, 24853d65, f5028c19), // pak9_genv99e.pk3 cgame + SHA(3969f078, 0d160315, 7679863d, 16a971ba, 11e3785c, 7174e64e, 3d0cf2a5, b241ae57), // pak9_genv99e.pk3 ui + SHA(cc2a6763, 0b473550, e0911ab9, d7a43c0a, 8cc58db9, bc03c901, b663ba85, a9820a81), // pak9_genv99e.pk3 qagame + + // CorkScrew Mod + SHA(0370caba, bb1245d8, 27190f6a, c944ae44, f7bb4964, 01b07012, c64ddaf9, 85eb6897), // CorkScrew_216.pk3 cgame + SHA(e805c1d5, 4a2b1fe2, 33f0bb02, 2e4c1a18, 98ad6248, a7a3101a, 1b233a11, 6f2290c7), // CorkScrew_216.pk3 ui + SHA(aaa3c7df, 8656d11d, 8caae055, b09a0759, 4f99f840, 410e6848, 015d5fe2, 6a498d9c), // CorkScrew_216.pk3 qagame +}; + +qboolean fs_check_trusted_vm_hash(unsigned char *hash) +{ + // Returns qtrue if hash is trusted, qfalse otherwise + int i; + unsigned int int_hash[8]; + for (i = 0; i < 8; ++i) + int_hash[i] = (((unsigned int)(hash[4 * i])) << 24) + (((unsigned int)(hash[4 * i + 1])) << 16) + + (((unsigned int)(hash[4 * i + 2])) << 8) + ((unsigned int)(hash[4 * i + 3])); + for (i = 0; i < ARRAY_LEN(trusted_vms); ++i) + if (!memcmp(int_hash, trusted_vms[i], 32)) + return qtrue; + return qfalse; +} + +#endif // NEW_FILESYSTEM diff --git a/src/filesystem/fscore/fsc_cache.cpp b/src/filesystem/fscore/fsc_cache.cpp new file mode 100644 index 000000000..b036cc87d --- /dev/null +++ b/src/filesystem/fscore/fsc_cache.cpp @@ -0,0 +1,535 @@ +/* +=========================================================================== +Copyright (C) 2017 Noah Metzger (chomenor@gmail.com) + +This file is part of Quake III Arena source code. + +Quake III Arena source code is free software; you can redistribute it +and/or modify it under the terms of the GNU General Public License as +published by the Free Software Foundation; either version 2 of the License, +or (at your option) any later version. + +Quake III Arena source code is distributed in the hope that it will be +useful, but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with Quake III Arena source code; if not, write to the Free Software +Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA +=========================================================================== +*/ + +#ifdef NEW_FILESYSTEM +#include "fscore.h" + +#define STACKPTR_SRC(pointer) (FSC_STACK_RETRIEVE(&xw->source_fs->general_stack, pointer, 0)) // non-null +#define STACKPTRN_SRC(pointer) (FSC_STACK_RETRIEVE(&xw->source_fs->general_stack, pointer, 1)) // null allowed + +/* ******************************************************************************** */ +// Support Structures +/* ******************************************************************************** */ + +typedef struct { + fsc_filesystem_t *source_fs; + + // Temporary - not written to file + fsc_stack_t filemap_stack; + fsc_hashtable_t filemap; + + // Generated by various export functions + fsc_stack_t export_stack; + fsc_hashtable_t export_string_repository; + + // Generated by fscache_export_filesystem + fsc_hashtable_t export_files; + fsc_hashtable_t export_directories; + + // Generated by fscache_export_shaders + fsc_hashtable_t export_shaders; + + // Generated by fscache_export_crosshairs + fsc_hashtable_t export_crosshairs; + + // Generated by fscache_build_pk3_hash_lookup + fsc_hashtable_t export_pk3_hash_lookup; +} export_work_t; + +typedef struct { + // Hash table compliance + fsc_hashtable_entry_t hte; + + fsc_stackptr_t source_file_ptr; + fsc_stackptr_t export_file_ptr; +} filemap_entry_t; + +/* ******************************************************************************** */ +// Data Building +/* ******************************************************************************** */ + +unsigned int integer_hash(unsigned int value) { return value; } + +fsc_stackptr_t convert_string(fsc_stackptr_t source_string, export_work_t *xw) +{ + // Converts a string from the standard string repository to the export repository + if (!source_string) + return 0; + return fsc_string_repository_getstring( + (const char *)STACKPTR_SRC(source_string), 1, &xw->export_string_repository, &xw->export_stack); +} + +fsc_stackptr_t file_map_lookup(fsc_stackptr_t source_file_ptr, export_work_t *xw) +{ + // Maps stackptr in general stack to converted file stackptr in export stack + unsigned int hash = integer_hash(source_file_ptr); + fsc_hashtable_iterator_t hti; + fsc_stackptr_t filemap_entry_ptr; + filemap_entry_t *filemap_entry; + + fsc_hashtable_open(&xw->filemap, hash, &hti); + while ((filemap_entry_ptr = fsc_hashtable_next(&hti))) + { + filemap_entry = (filemap_entry_t *)FSC_STACK_RETRIEVE(&xw->filemap_stack, filemap_entry_ptr, 0); + if (filemap_entry->source_file_ptr == source_file_ptr) + return filemap_entry->export_file_ptr; + } + + return 0; +} + +void file_map_insert(fsc_stackptr_t source_file_ptr, fsc_stackptr_t export_file_ptr, export_work_t *xw) +{ + unsigned int hash = integer_hash(source_file_ptr); + fsc_stackptr_t filemap_entry_ptr; + filemap_entry_t *filemap_entry; + + filemap_entry_ptr = fsc_stack_allocate(&xw->filemap_stack, sizeof(filemap_entry_t)); + filemap_entry = (filemap_entry_t *)FSC_STACK_RETRIEVE(&xw->filemap_stack, filemap_entry_ptr, 0); + + filemap_entry->source_file_ptr = source_file_ptr; + filemap_entry->export_file_ptr = export_file_ptr; + fsc_hashtable_insert(filemap_entry_ptr, hash, &xw->filemap); +} + +fsc_stackptr_t convert_file(fsc_stackptr_t source_file_ptr, export_work_t *xw) +{ + // Converts file from source filesystem to export filesystem if not already converted + // If file is already converted, just returns the existing converted file + // Should only be called for direct or pk3 sourcetype files (not custom sourcetypes) + fsc_file_t *source_file = (fsc_file_t *)STACKPTR_SRC(source_file_ptr); + fsc_stackptr_t export_file_ptr; + fsc_file_t *export_file; + int file_size = 0; + + // Check if the file has already been converted + export_file_ptr = file_map_lookup(source_file_ptr, xw); + if (export_file_ptr) + return export_file_ptr; + + // Get file structure size for copy + if (source_file->sourcetype == FSC_SOURCETYPE_DIRECT) + file_size = sizeof(fsc_file_direct_t); + else if (source_file->sourcetype == FSC_SOURCETYPE_PK3) + file_size = sizeof(fsc_file_frompk3_t); + else + fsc_fatal_error("convert_file called on unsupported sourcetype"); + + // Generate copy of file in export_stack + export_file_ptr = fsc_stack_allocate(&xw->export_stack, file_size); + export_file = (fsc_file_t *)FSC_STACK_RETRIEVE(&xw->export_stack, export_file_ptr, 0); + fsc_memcpy(export_file, source_file, file_size); + + // Reallocate stackptrs from original stack to export stack and other adjustments + export_file->qp_dir_ptr = convert_string(export_file->qp_dir_ptr, xw); + export_file->qp_name_ptr = convert_string(export_file->qp_name_ptr, xw); + export_file->qp_ext_ptr = convert_string(export_file->qp_ext_ptr, xw); + if (export_file->contents_cache) + { + export_file->contents_cache = fsc_stack_allocate(&xw->export_stack, source_file->filesize); + fsc_memcpy(FSC_STACK_RETRIEVE(&xw->export_stack, export_file->contents_cache, 0), + STACKPTR_SRC(source_file->contents_cache), source_file->filesize); + } + if (source_file->sourcetype == FSC_SOURCETYPE_DIRECT) + { + fsc_file_direct_t *export_file_typed = (fsc_file_direct_t *)export_file; + export_file_typed->qp_mod_ptr = convert_string(export_file_typed->qp_mod_ptr, xw); + export_file_typed->pk3dir_ptr = convert_string(export_file_typed->pk3dir_ptr, xw); + export_file_typed->refresh_count = 0; + export_file_typed->source_dir_id = 0; + // We can leave this null and let fsc_load_file patch it back in + // It checks enough data already (qpath, size, timestamp, etc.) to reasonably assume + // it is getting at least an identical file to the one cached + export_file_typed->os_path_ptr = 0; + } + if (source_file->sourcetype == FSC_SOURCETYPE_PK3) + { + fsc_file_frompk3_t *export_file_typed = (fsc_file_frompk3_t *)export_file; + export_file_typed->source_pk3 = convert_file(export_file_typed->source_pk3, xw); + } + + // Insert file into filemap, hash table, and iteration system + file_map_insert(source_file_ptr, export_file_ptr, xw); + fsc_hashtable_insert(export_file_ptr, + fsc_string_hash( + (const char *)STACKPTR_SRC(source_file->qp_name_ptr), (const char *)STACKPTRN_SRC(source_file->qp_dir_ptr)), + &xw->export_files); + fsc_iteration_register_file( + export_file_ptr, &xw->export_directories, &xw->export_string_repository, &xw->export_stack); + + return export_file_ptr; +} + +void fscache_build_filesystem(export_work_t *xw) +{ + int i; + fsc_hashtable_iterator_t hti; + fsc_stackptr_t source_file_ptr; + + // Estimate a reasonable hashtable size + int hashtable_target_size = xw->source_fs->active_stats.cacheable_file_count; + if (hashtable_target_size < 16384) + hashtable_target_size = 16384; + + // Initialize temporary filemap + fsc_stack_initialize(&xw->filemap_stack); + fsc_hashtable_initialize(&xw->filemap, &xw->filemap_stack, hashtable_target_size); + + // Initialize export structures + fsc_stack_initialize(&xw->export_stack); + fsc_hashtable_initialize(&xw->export_string_repository, &xw->export_stack, hashtable_target_size); + fsc_hashtable_initialize(&xw->export_files, &xw->export_stack, hashtable_target_size); + fsc_hashtable_initialize(&xw->export_directories, &xw->export_stack, hashtable_target_size / 4); + + // Iterate and process all files in filesystem + for (i = 0; i < xw->source_fs->files.bucket_count; ++i) + { + fsc_hashtable_open(&xw->source_fs->files, i, &hti); + while ((source_file_ptr = fsc_hashtable_next(&hti))) + { + fsc_file_t *source_file = (fsc_file_t *)STACKPTR_SRC(source_file_ptr); + + // Process active files from pk3 sources, or from direct sources if contents_cache is set + // Files referenced by shaders or crosshairs will be cached in their own cycle below, + // if not already cached here + if (!fsc_is_file_enabled(source_file, xw->source_fs)) + continue; + if (source_file->sourcetype == FSC_SOURCETYPE_PK3 || + (source_file->sourcetype == FSC_SOURCETYPE_DIRECT && source_file->contents_cache)) + { + convert_file(source_file_ptr, xw); + } + } + } +} + +void fscache_build_shaders(export_work_t *xw) +{ + int i; + fsc_hashtable_iterator_t hti; + fsc_stackptr_t source_shader_ptr; + unsigned int hash; + + fsc_stackptr_t export_shader_ptr; + fsc_shader_t *export_shader; + + // Estimate a reasonable hashtable size + int hashtable_target_size = xw->source_fs->active_stats.shader_count; + if (hashtable_target_size < 16384) + hashtable_target_size = 16384; + + // Initialize export structure + fsc_hashtable_initialize(&xw->export_shaders, &xw->export_stack, hashtable_target_size); + + // Iterate and process all shaders + for (i = 0; i < xw->source_fs->shaders.bucket_count; ++i) + { + fsc_hashtable_open(&xw->source_fs->shaders, i, &hti); + while ((source_shader_ptr = fsc_hashtable_next(&hti))) + { + const fsc_shader_t *source_shader = (fsc_shader_t *)STACKPTR_SRC(source_shader_ptr); + const fsc_file_t *source_file = (const fsc_file_t *)STACKPTR_SRC(source_shader->source_file_ptr); + + // Process only active shaders from direct/pk3 sources + if (source_file->sourcetype != FSC_SOURCETYPE_DIRECT && source_file->sourcetype != FSC_SOURCETYPE_PK3) + continue; + if (!is_shader_enabled(xw->source_fs, source_shader)) + continue; + + // Allocate new shader + export_shader_ptr = fsc_stack_allocate(&xw->export_stack, sizeof(fsc_shader_t)); + export_shader = (fsc_shader_t *)FSC_STACK_RETRIEVE(&xw->export_stack, export_shader_ptr, 0); + fsc_memcpy(export_shader, source_shader, sizeof(fsc_shader_t)); + + // Convert stackptrs within the shader + export_shader->shader_name_ptr = convert_string(export_shader->shader_name_ptr, xw); + export_shader->source_file_ptr = convert_file(export_shader->source_file_ptr, xw); + + // Add shader to export hashtable + hash = fsc_string_hash((const char *)STACKPTR_SRC(source_shader->shader_name_ptr), 0); + fsc_hashtable_insert(export_shader_ptr, hash, &xw->export_shaders); + } + } +} + +void fscache_build_crosshairs(export_work_t *xw) +{ + int i; + fsc_hashtable_iterator_t hti; + fsc_stackptr_t source_crosshair_ptr; + + fsc_stackptr_t export_crosshair_ptr; + fsc_crosshair_t *export_crosshair; + + // Initialize export structure + fsc_hashtable_initialize(&xw->export_crosshairs, &xw->export_stack, 1); + + // Iterate and process all crosshairs + for (i = 0; i < xw->source_fs->crosshairs.bucket_count; ++i) + { + fsc_hashtable_open(&xw->source_fs->crosshairs, i, &hti); + while ((source_crosshair_ptr = fsc_hashtable_next(&hti))) + { + const fsc_crosshair_t *source_crosshair = (fsc_crosshair_t *)STACKPTR_SRC(source_crosshair_ptr); + const fsc_file_t *source_file = (const fsc_file_t *)STACKPTR_SRC(source_crosshair->source_file_ptr); + + // Process only active crosshairs from direct/pk3 sources + if (source_file->sourcetype != FSC_SOURCETYPE_DIRECT && source_file->sourcetype != FSC_SOURCETYPE_PK3) + continue; + if (!is_crosshair_enabled(xw->source_fs, source_crosshair)) + continue; + + // Allocate new crosshair + export_crosshair_ptr = fsc_stack_allocate(&xw->export_stack, sizeof(fsc_crosshair_t)); + export_crosshair = (fsc_crosshair_t *)FSC_STACK_RETRIEVE(&xw->export_stack, export_crosshair_ptr, 0); + fsc_memcpy(export_crosshair, source_crosshair, sizeof(fsc_crosshair_t)); + + // Convert stackptrs within the crosshair + export_crosshair->source_file_ptr = convert_file(export_crosshair->source_file_ptr, xw); + + // Add crosshair to export hashtable + fsc_hashtable_insert(export_crosshair_ptr, source_crosshair->hash, &xw->export_crosshairs); + } + } +} + +void fscache_build_pk3_hash_lookup(export_work_t *xw) +{ + int i; + fsc_hashtable_iterator_t hti; + fsc_stackptr_t source_entry_ptr; + + // Estimate a reasonable hashtable size + int hashtable_target_size = xw->source_fs->active_stats.valid_pk3_count; + if (hashtable_target_size < 1024) + hashtable_target_size = 1024; + + // Initialize export structure + fsc_hashtable_initialize(&xw->export_pk3_hash_lookup, &xw->export_stack, hashtable_target_size); + + // Iterate and process all entries + for (i = 0; i < xw->source_fs->pk3_hash_lookup.bucket_count; ++i) + { + fsc_hashtable_open(&xw->source_fs->pk3_hash_lookup, i, &hti); + while ((source_entry_ptr = fsc_hashtable_next(&hti))) + { + fsc_stackptr_t output_pk3_file; + fsc_pk3_hash_map_entry_t *source_entry = (fsc_pk3_hash_map_entry_t *)STACKPTR_SRC(source_entry_ptr); + fsc_file_direct_t *source_pk3_file = (fsc_file_direct_t *)STACKPTR_SRC(source_entry->pk3); + if (!fsc_is_file_enabled((fsc_file_t *)source_pk3_file, xw->source_fs)) + continue; + + output_pk3_file = convert_file(source_entry->pk3, xw); + register_pk3_hash_lookup_entry(output_pk3_file, &xw->export_pk3_hash_lookup, &xw->export_stack); + } + } +} + +/* ******************************************************************************** */ +// Main +/* ******************************************************************************** */ + +static int fsc_cache_export_data(fsc_filesystem_t *source_fs, fsc_stream_t *stream) +{ + // Returns 1 on error, 0 on success. stream->data must be freed by caller on success. + int error_state = 0; + export_work_t xw; + xw.source_fs = source_fs; + + // Build export data + fscache_build_filesystem(&xw); + fscache_build_shaders(&xw); + fscache_build_crosshairs(&xw); + fscache_build_pk3_hash_lookup(&xw); + + // Free filemap + fsc_stack_free(&xw.filemap_stack); + fsc_hashtable_free(&xw.filemap); + + // Set up export stream + stream->size = + fsc_stack_get_export_size(&xw.export_stack) + fsc_hashtable_get_export_size(&xw.export_string_repository) + + fsc_hashtable_get_export_size(&xw.export_files) + fsc_hashtable_get_export_size(&xw.export_directories) + + fsc_hashtable_get_export_size(&xw.export_shaders) + fsc_hashtable_get_export_size(&xw.export_crosshairs) + + fsc_hashtable_get_export_size(&xw.export_pk3_hash_lookup); + stream->data = (char *)fsc_malloc(stream->size); + stream->position = 0; + + // Write export data to stream + if (!error_state) + if (fsc_stack_export(&xw.export_stack, stream)) + error_state = 1; + if (!error_state) + if (fsc_hashtable_export(&xw.export_string_repository, stream)) + error_state = 1; + if (!error_state) + if (fsc_hashtable_export(&xw.export_files, stream)) + error_state = 1; + if (!error_state) + if (fsc_hashtable_export(&xw.export_directories, stream)) + error_state = 1; + if (!error_state) + if (fsc_hashtable_export(&xw.export_shaders, stream)) + error_state = 1; + if (!error_state) + if (fsc_hashtable_export(&xw.export_crosshairs, stream)) + error_state = 1; + if (!error_state) + if (fsc_hashtable_export(&xw.export_pk3_hash_lookup, stream)) + error_state = 1; + + // Free export data + fsc_stack_free(&xw.export_stack); + fsc_hashtable_free(&xw.export_string_repository); + fsc_hashtable_free(&xw.export_files); + fsc_hashtable_free(&xw.export_directories); + fsc_hashtable_free(&xw.export_shaders); + fsc_hashtable_free(&xw.export_crosshairs); + fsc_hashtable_free(&xw.export_pk3_hash_lookup); + + if (error_state) + fsc_free(stream->data); + return error_state; +} + +static int fsc_cache_import_data(fsc_stream_t *stream, fsc_filesystem_t *target_fs) +{ + // Returns 1 on error, 0 on success. + fsc_memset(target_fs, 0, sizeof(*target_fs)); + + if (fsc_stack_import(&target_fs->general_stack, stream)) + goto error; + if (fsc_hashtable_import(&target_fs->string_repository, &target_fs->general_stack, stream)) + goto error; + if (fsc_hashtable_import(&target_fs->files, &target_fs->general_stack, stream)) + goto error; + if (fsc_hashtable_import(&target_fs->directories, &target_fs->general_stack, stream)) + goto error; + if (fsc_hashtable_import(&target_fs->shaders, &target_fs->general_stack, stream)) + goto error; + if (fsc_hashtable_import(&target_fs->crosshairs, &target_fs->general_stack, stream)) + goto error; + if (fsc_hashtable_import(&target_fs->pk3_hash_lookup, &target_fs->general_stack, stream)) + goto error; + if (stream->size != stream->position) + goto error; + + // Start refresh count at 2 instead of 1 so files will go into the "new" + // stats category on the first refresh + target_fs->refresh_count = 2; + return 0; + +error: + fsc_filesystem_free(target_fs); + return 1; +} + +typedef struct { + unsigned int version; + unsigned int size; +} fscache_header_t; + +int fsc_cache_export_file(fsc_filesystem_t *source_fs, void *os_path, fsc_errorhandler_t *eh) +{ + // Returns 1 on error, 0 on success. + fscache_header_t header; + fsc_stream_t stream; + void *fp; + + if (fsc_cache_export_data(source_fs, &stream)) + { + fsc_report_error(eh, FSC_ERROR_GENERAL, "error generating cache file data", 0); + return 1; + } + + // Open the output file and write header + fp = fsc_open_file(os_path, "wb"); + if (!fp) + { + fsc_free(stream.data); + fsc_report_error(eh, FSC_ERROR_GENERAL, "failed to open output file", 0); + return 1; + } + header.version = FSC_CACHE_VERSION; + header.size = stream.position; + fsc_fwrite(&header, sizeof(header), fp); + + // Write the data + fsc_fwrite(stream.data, header.size, fp); + + // Close file and free data + fsc_fclose(fp); + fsc_free(stream.data); + return 0; +} + +int fsc_cache_import_file(void *os_path, fsc_filesystem_t *target_fs, fsc_errorhandler_t *eh) +{ + // Returns 1 on error, 0 on success. On error filesystem will not be initialized. + void *fp; + fscache_header_t header; + fsc_stream_t stream; + + // Open the input file and read header + fp = fsc_open_file(os_path, "rb"); + if (!fp) + { + fsc_report_error(eh, FSC_ERROR_GENERAL, "failed to open input file", 0); + return 1; + } + if (fsc_fread(&header, sizeof(header), fp) != sizeof(header)) + { + fsc_report_error(eh, FSC_ERROR_GENERAL, "failed to read cache file header", 0); + fsc_fclose(fp); + return 1; + } + if (header.version != FSC_CACHE_VERSION) + { + fsc_report_error(eh, FSC_ERROR_GENERAL, "cache file has wrong version", 0); + fsc_fclose(fp); + return 1; + } + + // Read data and close file + stream.data = (char *)fsc_malloc(header.size); + if (fsc_fread(stream.data, header.size, fp) != header.size) + { + fsc_report_error(eh, FSC_ERROR_GENERAL, "error reading cache file data", 0); + return 1; + } + fsc_fclose(fp); + + // Load data into filesystem + stream.position = 0; + stream.size = header.size; + if (fsc_cache_import_data(&stream, target_fs)) + { + fsc_report_error(eh, FSC_ERROR_GENERAL, "error loading cache data", 0); + return 1; + } + + // Free the data + fsc_free(stream.data); + return 0; +} + +#endif // NEW_FILESYSTEM diff --git a/src/filesystem/fscore/fsc_crosshair.cpp b/src/filesystem/fscore/fsc_crosshair.cpp new file mode 100644 index 000000000..7140c9370 --- /dev/null +++ b/src/filesystem/fscore/fsc_crosshair.cpp @@ -0,0 +1,71 @@ +/* +=========================================================================== +Copyright (C) 2017 Noah Metzger (chomenor@gmail.com) + +This file is part of Quake III Arena source code. + +Quake III Arena source code is free software; you can redistribute it +and/or modify it under the terms of the GNU General Public License as +published by the Free Software Foundation; either version 2 of the License, +or (at your option) any later version. + +Quake III Arena source code is distributed in the hope that it will be +useful, but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with Quake III Arena source code; if not, write to the Free Software +Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA +=========================================================================== +*/ + +#ifdef NEW_FILESYSTEM +#include "fscore.h" + +// This section is to support future crosshair lookup features + +#define STACKPTR(pointer) (FSC_STACK_RETRIEVE(&fs->general_stack, pointer, 0)) // non-null + +/* ******************************************************************************** */ +// Crosshair Indexing +/* ******************************************************************************** */ + +int index_crosshair(fsc_filesystem_t *fs, fsc_stackptr_t source_file_ptr, fsc_errorhandler_t *eh) +{ + // Returns 1 on success, 0 otherwise. + fsc_file_t *source_file = (fsc_file_t *)STACKPTR(source_file_ptr); + unsigned int hash; + + char *data = fsc_extract_file_allocated(fs, source_file, 0); + if (!data) + { + fsc_report_error(eh, FSC_ERROR_CROSSHAIRFILE, "failed to extract/open crosshair file", source_file); + return 0; + } + + hash = fsc_block_checksum(data, source_file->filesize); + fsc_free(data); + + { + fsc_stackptr_t new_crosshair_ptr = fsc_stack_allocate(&fs->general_stack, sizeof(fsc_crosshair_t)); + fsc_crosshair_t *new_crosshair = (fsc_crosshair_t *)STACKPTR(new_crosshair_ptr); + + new_crosshair->hash = hash; + new_crosshair->source_file_ptr = source_file_ptr; + fsc_hashtable_insert(new_crosshair_ptr, hash, &fs->crosshairs); + } + + return 1; +} + +/* ******************************************************************************** */ +// Other +/* ******************************************************************************** */ + +int is_crosshair_enabled(fsc_filesystem_t *fs, const fsc_crosshair_t *crosshair) +{ + return fsc_is_file_enabled((const fsc_file_t *)STACKPTR(crosshair->source_file_ptr), fs); +} + +#endif // NEW_FILESYSTEM diff --git a/src/filesystem/fscore/fsc_gameparse.cpp b/src/filesystem/fscore/fsc_gameparse.cpp new file mode 100644 index 000000000..ec0ab1f39 --- /dev/null +++ b/src/filesystem/fscore/fsc_gameparse.cpp @@ -0,0 +1,225 @@ +/* +=========================================================================== +Copyright (C) 1999-2005 Id Software, Inc. + +This file is part of Quake III Arena source code. + +Quake III Arena source code is free software; you can redistribute it +and/or modify it under the terms of the GNU General Public License as +published by the Free Software Foundation; either version 2 of the License, +or (at your option) any later version. + +Quake III Arena source code is distributed in the hope that it will be +useful, but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with Quake III Arena source code; if not, write to the Free Software +Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA +=========================================================================== +*/ + +#ifdef NEW_FILESYSTEM +#include "fscore.h" + +/* ******************************************************************************** */ +// Support functions from qcommon.c +/* ******************************************************************************** */ + +#define qboolean int +#define qtrue 1 +#define qfalse 0 +#define NULL 0 + +#define FSC_EDIT + +#ifdef FSC_EDIT +void fsc_SkipRestOfLine(char **data) +{ +#else +void SkipRestOfLine(char **data) +{ +#endif + char *p; + int c; + + p = *data; + while ((c = *p++) != 0) + { + if (c == '\n') + { +#ifndef FSC_EDIT + com_lines++; +#endif + break; + } + } + + *data = p; +} + +#ifdef FSC_EDIT +static char *fsc_SkipWhitespace(char *data, qboolean *hasNewLines) +{ +#else +static char *SkipWhitespace(char *data, qboolean *hasNewLines) +{ +#endif + int c; + + while ((c = *data) <= ' ') + { + if (!c) + { + return NULL; + } + if (c == '\n') + { +#ifndef FSC_EDIT + com_lines++; +#endif + *hasNewLines = qtrue; + } + data++; + } + + return data; +} + +#ifdef FSC_EDIT +// com_token is a buffer of length MAX_TOKEN_CHARS that stores the token being returned. +char *fsc_COM_ParseExt(char *com_token, char **data_p, int allowLineBreaks) +#else +char *COM_ParseExt(char **data_p, qboolean allowLineBreaks) +#endif +{ + int c = 0, len; + qboolean hasNewLines = qfalse; + char *data; + + data = *data_p; + len = 0; + com_token[0] = 0; + + // make sure incoming data is valid + if (!data) + { + *data_p = NULL; + return com_token; + } + + while (1) + { + // skip whitespace + data = fsc_SkipWhitespace(data, &hasNewLines); + if (!data) + { + *data_p = NULL; + return com_token; + } + if (hasNewLines && !allowLineBreaks) + { + *data_p = data; + return com_token; + } + + c = *data; + + // skip double slash comments + if (c == '/' && data[1] == '/') + { + data += 2; + while (*data && *data != '\n') + { + data++; + } + } + // skip /* */ comments + else if (c == '/' && data[1] == '*') + { + data += 2; + while (*data && (*data != '*' || data[1] != '/')) + { + data++; + } + if (*data) + { + data += 2; + } + } + else + { + break; + } + } + + // handle quoted strings + if (c == '\"') + { + data++; + while (1) + { + c = *data++; + if (c == '\"' || !c) + { + com_token[len] = 0; + *data_p = (char *)data; + return com_token; + } + if (len < FSC_MAX_TOKEN_CHARS - 1) + { + com_token[len] = c; + len++; + } + } + } + + // parse a regular word + do + { + if (len < FSC_MAX_TOKEN_CHARS - 1) + { + com_token[len] = c; + len++; + } + data++; + c = *data; +#ifndef FSC_EDIT + if (c == '\n') + com_lines++; +#endif + } while (c > 32); + + com_token[len] = 0; + + *data_p = (char *)data; + return com_token; +} + +// Slightly modified from the original q_shared.c version +// depth parameter is initial depth (0 if expecting first brace) +int fsc_SkipBracedSection(char **program, int depth) +{ + char token[FSC_MAX_TOKEN_CHARS]; + + do + { + fsc_COM_ParseExt(token, program, qtrue); + if (token[1] == 0) + { + if (token[0] == '{') + { + depth++; + } + else if (token[0] == '}') + { + depth--; + } + } + } while (depth && *program); + + return depth; +} + +#endif // NEW_FILESYSTEM diff --git a/src/filesystem/fscore/fsc_iteration.cpp b/src/filesystem/fscore/fsc_iteration.cpp new file mode 100644 index 000000000..e14ddda25 --- /dev/null +++ b/src/filesystem/fscore/fsc_iteration.cpp @@ -0,0 +1,135 @@ +/* +=========================================================================== +Copyright (C) 2017 Noah Metzger (chomenor@gmail.com) + +This file is part of Quake III Arena source code. + +Quake III Arena source code is free software; you can redistribute it +and/or modify it under the terms of the GNU General Public License as +published by the Free Software Foundation; either version 2 of the License, +or (at your option) any later version. + +Quake III Arena source code is distributed in the hope that it will be +useful, but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with Quake III Arena source code; if not, write to the Free Software +Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA +=========================================================================== +*/ + +#ifdef NEW_FILESYSTEM +#include "fscore.h" + +// This section is used to provide faster file iteration when starting at a certain +// directory, rather than iterating over the entire filesystem. + +#define STACKPTR_LCL(pointer) (FSC_STACK_RETRIEVE(stack, pointer, 0)) // non-null, local stack parameter +#define STACKPTRN_LCL(pointer) (FSC_STACK_RETRIEVE(stack, pointer, 1)) // null allowed, local stack parameter + +/* ******************************************************************************** */ +// File Registration +/* ******************************************************************************** */ + +static int get_parent_qp_dir(const char *qp_dir, char *target) +{ + // Converts qp_dir string to parent dir + // In other words, strip everything including and after the last slash + // Target should be size FSC_MAX_QPATH + // Returns 0 if no parent directory, 1 otherwise + const char *current = qp_dir; + const char *last_slash = 0; + unsigned int length; + + while (*current) + { + if (*current == '/') + last_slash = current; + ++current; + } + + // If no slash is found, parent directory is root directory. Leave output null. + if (!last_slash) + return 0; + + // Copy in everything up to the slash + length = last_slash - qp_dir; + if (length > FSC_MAX_QPATH - 1) + length = FSC_MAX_QPATH - 1; + fsc_memcpy(target, qp_dir, length); + target[length] = 0; + return 1; +} + +static fsc_stackptr_t get_directory( + const char *qp_dir, fsc_hashtable_t *directories, fsc_hashtable_t *string_repository, fsc_stack_t *stack) +{ + // Null qp_dir represents the root directory + unsigned int qp_dir_hash = qp_dir ? fsc_string_hash(qp_dir, 0) : 0; + + fsc_hashtable_iterator_t hti; + fsc_stackptr_t directory_ptr; + fsc_directory_t *directory; + + char parent_qp_dir[FSC_MAX_QPATH]; + int parent_path_found; + fsc_stackptr_t parent_dir_ptr; + fsc_directory_t *parent_dir; + + // Check if directory is already in the hash table + fsc_hashtable_open(directories, qp_dir_hash, &hti); + while ((directory_ptr = fsc_hashtable_next(&hti))) + { + directory = (fsc_directory_t *)STACKPTR_LCL(directory_ptr); + if (!directory->qp_dir_ptr) + { + if (!qp_dir) + return directory_ptr; + continue; + } + if (!qp_dir) + continue; + if (!fsc_stricmp((const char *)STACKPTR_LCL(directory->qp_dir_ptr), qp_dir)) + return directory_ptr; + } + + // It isn't, so create a new directory + directory_ptr = fsc_stack_allocate(stack, sizeof(fsc_directory_t)); + directory = (fsc_directory_t *)STACKPTR_LCL(directory_ptr); + directory->qp_dir_ptr = qp_dir ? fsc_string_repository_getstring(qp_dir, 1, string_repository, stack) : 0; + fsc_hashtable_insert(directory_ptr, qp_dir_hash, directories); + + // If we are at the root directory, don't proceed any further + if (!qp_dir) + return directory_ptr; + + // Get the parent directory + parent_path_found = get_parent_qp_dir(qp_dir, parent_qp_dir); + parent_dir_ptr = get_directory(parent_path_found ? parent_qp_dir : 0, directories, string_repository, stack); + parent_dir = (fsc_directory_t *)STACKPTR_LCL(parent_dir_ptr); + + // Add current directory to parent's sub_directory linked list + directory->peer_directory = parent_dir->sub_directory; + parent_dir->sub_directory = directory_ptr; + + return directory_ptr; +} + +void fsc_iteration_register_file( + fsc_stackptr_t file_ptr, fsc_hashtable_t *directories, fsc_hashtable_t *string_repository, fsc_stack_t *stack) +{ + fsc_file_t *file = (fsc_file_t *)STACKPTR_LCL(file_ptr); + + // Get directory + fsc_stackptr_t directory_ptr = + get_directory((const char *)STACKPTRN_LCL(file->qp_dir_ptr), directories, string_repository, stack); + fsc_directory_t *directory = (fsc_directory_t *)STACKPTR_LCL(directory_ptr); + + // Add file to directory's sub_file linked list + file->next_in_directory = directory->sub_file; + directory->sub_file = file_ptr; +} + +#endif // NEW_FILESYSTEM diff --git a/src/filesystem/fscore/fsc_main.cpp b/src/filesystem/fscore/fsc_main.cpp new file mode 100644 index 000000000..e6bf768a8 --- /dev/null +++ b/src/filesystem/fscore/fsc_main.cpp @@ -0,0 +1,545 @@ +/* +=========================================================================== +Copyright (C) 2017 Noah Metzger (chomenor@gmail.com) + +This file is part of Quake III Arena source code. + +Quake III Arena source code is free software; you can redistribute it +and/or modify it under the terms of the GNU General Public License as +published by the Free Software Foundation; either version 2 of the License, +or (at your option) any later version. + +Quake III Arena source code is distributed in the hope that it will be +useful, but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with Quake III Arena source code; if not, write to the Free Software +Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA +=========================================================================== +*/ + +#ifdef NEW_FILESYSTEM +#include "fscore.h" + +#define STACKPTR(pointer) (FSC_STACK_RETRIEVE(&fs->general_stack, pointer, 0)) // non-null +#define STACKPTRN(pointer) (FSC_STACK_RETRIEVE(&fs->general_stack, pointer, 1)) // null allowed + +/* ******************************************************************************** */ +// Direct Sourcetype Operations +/* ******************************************************************************** */ + +static int fsc_direct_is_file_active(const fsc_file_t *file, const fsc_filesystem_t *fs) +{ + return ((fsc_file_direct_t *)file)->refresh_count == fs->refresh_count; +} + +static const char *fsc_direct_get_mod_dir(const fsc_file_t *file, const fsc_filesystem_t *fs) +{ + return (const char *)STACKPTR(((fsc_file_direct_t *)file)->qp_mod_ptr); +} + +static int fsc_direct_extract_data( + const fsc_file_t *file, char *buffer, const fsc_filesystem_t *fs, fsc_errorhandler_t *eh) +{ + void *fp; + unsigned int result; + + // Open the file + fp = fsc_open_file(STACKPTR(((fsc_file_direct_t *)file)->os_path_ptr), "rb"); + if (!fp) + { + fsc_report_error(eh, FSC_ERROR_EXTRACT, "failed to open file", 0); + return 1; + } + + result = fsc_fread(buffer, file->filesize, fp); + if (result != file->filesize) + { + fsc_report_error(eh, FSC_ERROR_EXTRACT, "failed to read all data from file", 0); + fsc_fclose(fp); + return 1; + } + + fsc_fclose(fp); + return 0; +} + +static fsc_sourcetype_t direct_sourcetype = { + FSC_SOURCETYPE_DIRECT, fsc_direct_is_file_active, fsc_direct_get_mod_dir, fsc_direct_extract_data}; + +/* ******************************************************************************** */ +// Common file operations +/* ******************************************************************************** */ + +static const fsc_sourcetype_t *fsc_get_sourcetype(const fsc_file_t *file, const fsc_filesystem_t *fs) +{ + int i; + + // Check built in sourcetypes + if (file->sourcetype == FSC_SOURCETYPE_DIRECT) + return &direct_sourcetype; + if (file->sourcetype == FSC_SOURCETYPE_PK3) + return &pk3_sourcetype; + + // Check custom sourcetypes + for (i = 0; i < FSC_CUSTOM_SOURCETYPE_COUNT; ++i) + { + if (file->sourcetype == fs->custom_sourcetypes[i].sourcetype_id) + return &fs->custom_sourcetypes[i]; + } + + return 0; +} + +const fsc_file_direct_t *fsc_get_base_file(const fsc_file_t *file, const fsc_filesystem_t *fs) +{ + // Returns the source pk3 if file is from a pk3, the file itself if file is on disk, + // and null if the file is from a custom sourcetype + FSC_ASSERT(file); + if (file->sourcetype == FSC_SOURCETYPE_DIRECT) + { + return (const fsc_file_direct_t *)file; + } + if (file->sourcetype == FSC_SOURCETYPE_PK3) + { + return (fsc_file_direct_t *)STACKPTR(((fsc_file_frompk3_t *)file)->source_pk3); + } + return 0; +} + +int fsc_extract_file(const fsc_file_t *file, char *buffer, const fsc_filesystem_t *fs, fsc_errorhandler_t *eh) +{ + // Provided buffer should be size file->filesize + // Returns 0 on success, 1 on error + const fsc_sourcetype_t *sourcetype; + if (file->contents_cache) + { + fsc_memcpy(buffer, STACKPTR(file->contents_cache), file->filesize); + return 0; + } + sourcetype = fsc_get_sourcetype(file, fs); + FSC_ASSERT(sourcetype && sourcetype->extract_data); + return sourcetype->extract_data(file, buffer, fs, eh); +} + +int fsc_is_file_enabled(const fsc_file_t *file, const fsc_filesystem_t *fs) +{ + // Returns 1 if file enabled, 0 otherwise + const fsc_sourcetype_t *sourcetype = fsc_get_sourcetype(file, fs); + FSC_ASSERT(sourcetype && sourcetype->is_file_active); + return sourcetype->is_file_active(file, fs); +} + +const char *fsc_get_mod_dir(const fsc_file_t *file, const fsc_filesystem_t *fs) +{ + // Should not return null; may return empty string to represent omitted/invalid mod directory + const fsc_sourcetype_t *sourcetype = fsc_get_sourcetype(file, fs); + const char *mod_directory; + FSC_ASSERT(sourcetype && sourcetype->get_mod_dir); + mod_directory = sourcetype->get_mod_dir(file, fs); + FSC_ASSERT(mod_directory); + return mod_directory; +} + +char *fsc_extract_file_allocated(fsc_filesystem_t *fs, fsc_file_t *file, fsc_errorhandler_t *eh) +{ + // Returns data on success, 0 on failure + // Caller is responsible for calling fsc_free on extracted data + char *data; + if (file->filesize + 1 < file->filesize) + return 0; + data = (char *)fsc_malloc(file->filesize + 1); + if (fsc_extract_file(file, data, fs, eh)) + { + fsc_free(data); + return 0; + } + data[file->filesize] = 0; + return data; +} + +/* ******************************************************************************** */ +// File to printable string functions +/* ******************************************************************************** */ + +#define FSC_ADD_STRING(string) fsc_stream_append_string(stream, string) + +void fsc_file_to_stream( + const fsc_file_t *file, fsc_stream_t *stream, const fsc_filesystem_t *fs, int include_mod, int include_pk3_origin) +{ + if (include_mod) + { + const char *mod_dir = fsc_get_mod_dir(file, fs); + if (!*mod_dir) + mod_dir = ""; + FSC_ADD_STRING(mod_dir); + FSC_ADD_STRING("/"); + } + + if (include_pk3_origin) + { + if (file->sourcetype == FSC_SOURCETYPE_DIRECT && ((fsc_file_direct_t *)file)->pk3dir_ptr) + { + FSC_ADD_STRING((const char *)STACKPTR(((fsc_file_direct_t *)file)->pk3dir_ptr)); + FSC_ADD_STRING(".pk3dir->"); + } + else if (file->sourcetype == FSC_SOURCETYPE_PK3) + { + fsc_file_to_stream((const fsc_file_t *)fsc_get_base_file(file, fs), stream, fs, 0, 0); + FSC_ADD_STRING("->"); + } + } + + if (file->qp_dir_ptr) + { + FSC_ADD_STRING((const char *)STACKPTR(file->qp_dir_ptr)); + FSC_ADD_STRING("/"); + } + FSC_ADD_STRING((const char *)STACKPTR(file->qp_name_ptr)); + if (file->qp_ext_ptr) + { + FSC_ADD_STRING("."); + FSC_ADD_STRING((const char *)STACKPTR(file->qp_ext_ptr)); + } +} + +/* ******************************************************************************** */ +// File Indexing +/* ******************************************************************************** */ + +static void fsc_merge_stats(const fsc_stats_t *source, fsc_stats_t *target) +{ + target->valid_pk3_count += source->valid_pk3_count; + target->pk3_subfile_count += source->pk3_subfile_count; + target->shader_file_count += source->shader_file_count; + target->shader_count += source->shader_count; + target->total_file_count += source->total_file_count; + target->cacheable_file_count += source->cacheable_file_count; +} + +void fsc_register_file(fsc_stackptr_t file_ptr, fsc_filesystem_t *fs, fsc_errorhandler_t *eh) +{ + // Registers file in index and loads secondary content such as shaders + // Called for both files on disk and in pk3s + fsc_file_t *file = (fsc_file_t *)STACKPTR(file_ptr); + fsc_file_direct_t *base_file = (fsc_file_direct_t *)fsc_get_base_file(file, fs); + const char *qp_dir = (const char *)STACKPTRN(file->qp_dir_ptr); + const char *qp_name = (const char *)STACKPTR(file->qp_name_ptr); + const char *qp_ext = (const char *)STACKPTRN(file->qp_ext_ptr); + + // Register file for main lookup and directory iteration + fsc_hashtable_insert(file_ptr, fsc_string_hash(qp_name, qp_dir), &fs->files); + fsc_iteration_register_file(file_ptr, &fs->directories, &fs->string_repository, &fs->general_stack); + + // Index shaders and update shader counter on base file + if (qp_ext && qp_dir && !fsc_stricmp(qp_dir, "scripts") && !fsc_stricmp(qp_ext, "shader")) + { + int count = index_shader_file(fs, file_ptr, eh); + if (base_file) + { + base_file->shader_file_count += 1; + base_file->shader_count += count; + base_file->f.flags |= FSC_FILEFLAG_LINKED_CONTENT; + } + } + + // Index crosshairs + if (qp_dir && !fsc_stricmp(qp_dir, "gfx/2d")) + { + char buffer[10]; + fsc_strncpy(buffer, qp_name, sizeof(buffer)); + if (!fsc_stricmp(buffer, "crosshair")) + { + index_crosshair(fs, file_ptr, eh); + if (base_file) + base_file->f.flags |= FSC_FILEFLAG_LINKED_CONTENT; + } + } + + // Cache small arena and bot file contents + if (file->filesize < 16384 && qp_dir && !fsc_stricmp(qp_dir, "scripts") && qp_ext && + (!fsc_stricmp(qp_ext, "arena") || !fsc_stricmp(qp_ext, "bot"))) + { + char *source_data = fsc_extract_file_allocated(fs, file, eh); + if (source_data) + { + fsc_stackptr_t target_ptr = fsc_stack_allocate(&fs->general_stack, file->filesize); + char *target_data = (char *)STACKPTR(target_ptr); + fsc_memcpy(target_data, source_data, file->filesize); + file->contents_cache = target_ptr; + fsc_free(source_data); + } + } +} + +static int fsc_nstring_compare(const char *s1, const char *s2) +{ + // Compares two potentially null strings, returns 1 if matching, 0 otherwise + if (!s1 && !s2) + return 1; + if (!s1 || !s2) + return 0; + return !fsc_strcmp(s1, s2); +} + +void fsc_load_file(int source_dir_id, const void *os_path, const char *mod_dir, const char *pk3dir_name, + const char *qp_dir, const char *qp_name, const char *qp_ext, unsigned int os_timestamp, unsigned int filesize, + fsc_filesystem_t *fs, fsc_errorhandler_t *eh) +{ + fsc_stackptr_t file_ptr; + fsc_file_direct_t *file; + fsc_hashtable_iterator_t hti; + unsigned int fs_hash = fsc_string_hash(qp_name, qp_dir); + int unindexed_file = 0; // File was not present in the index at all + int new_file = 0; // File was not present in last refresh, but may have been in the index + + FSC_ASSERT(os_path); + FSC_ASSERT(qp_name); + FSC_ASSERT(fs); + + // Search filesystem to see if a sufficiently equivalent entry already exists + fsc_hashtable_open(&fs->files, fs_hash, &hti); + while ((file_ptr = fsc_hashtable_next(&hti))) + { + file = (fsc_file_direct_t *)STACKPTR(file_ptr); + if (file->f.sourcetype != FSC_SOURCETYPE_DIRECT) + continue; + if (!fsc_nstring_compare((char *)STACKPTR(file->f.qp_name_ptr), qp_name)) + continue; + if (!fsc_nstring_compare((char *)STACKPTRN(file->f.qp_dir_ptr), qp_dir)) + continue; + if (!fsc_nstring_compare((char *)STACKPTRN(file->f.qp_ext_ptr), qp_ext)) + continue; + if (!fsc_nstring_compare((char *)STACKPTRN(file->qp_mod_ptr), mod_dir)) + continue; + if (!fsc_nstring_compare((char *)STACKPTRN(file->pk3dir_ptr), pk3dir_name)) + continue; + if (file->os_path_ptr && fsc_compare_os_path(STACKPTR(file->os_path_ptr), os_path)) + continue; + if (file->f.filesize != filesize || file->os_timestamp != os_timestamp) + { + if (file->os_path_ptr && !(file->f.flags & FSC_FILEFLAG_LINKED_CONTENT) && !file->f.contents_cache) + { + // Reuse the same file object to save memory (this prevents files actively written + // by the game such as logs generating a new file object every refresh) + file->f.filesize = filesize; + file->os_timestamp = os_timestamp; + break; + } + else + { + // Otherwise treat the file as non-matching + continue; + } + } + break; + } + + if (file_ptr) + { + // Have existing entry + if (file->refresh_count == fs->refresh_count) + { + // Existing file already active. This can happen with if there are duplicate source directories + // loaded in the same refresh cycle. Just leave the existing file unchanged. + return; + } + + // Activate the entry + if (file->refresh_count != fs->refresh_count - 1) + new_file = 1; + file->refresh_count = fs->refresh_count; + } + + else + { + // Create a new entry + file_ptr = fsc_stack_allocate(&fs->general_stack, sizeof(*file)); + file = (fsc_file_direct_t *)STACKPTR(file_ptr); + + // Set up fields (other fields are zeroed by default due to stack allocation) + file->f.sourcetype = FSC_SOURCETYPE_DIRECT; + file->f.qp_dir_ptr = + qp_dir ? fsc_string_repository_getstring(qp_dir, 1, &fs->string_repository, &fs->general_stack) : 0; + file->f.qp_name_ptr = fsc_string_repository_getstring(qp_name, 1, &fs->string_repository, &fs->general_stack); + file->f.qp_ext_ptr = + qp_ext ? fsc_string_repository_getstring(qp_ext, 1, &fs->string_repository, &fs->general_stack) : 0; + file->qp_mod_ptr = + mod_dir ? fsc_string_repository_getstring(mod_dir, 1, &fs->string_repository, &fs->general_stack) : 0; + file->pk3dir_ptr = + pk3dir_name ? fsc_string_repository_getstring(pk3dir_name, 1, &fs->string_repository, &fs->general_stack) + : 0; + file->f.filesize = filesize; + file->os_timestamp = os_timestamp; + file->refresh_count = fs->refresh_count; + + unindexed_file = new_file = 1; + } + + // Update source dir and download folder flag + file->source_dir_id = source_dir_id; + file->f.flags &= ~FSC_FILEFLAG_DLPK3; + if (qp_ext && !fsc_stricmp(qp_ext, "pk3") && qp_dir && !fsc_stricmp(qp_dir, "downloads")) + { + file->f.flags |= FSC_FILEFLAG_DLPK3; + } + + // Save os path. This happens on loading a new file, and also when first activating an entry that was loaded from + // cache. + if (!file->os_path_ptr) + { + int os_path_size = fsc_os_path_size(os_path); + file->os_path_ptr = fsc_stack_allocate(&fs->general_stack, os_path_size); + fsc_memcpy(STACKPTR(file->os_path_ptr), os_path, os_path_size); + } + + // Register file and load contents + if (unindexed_file) + { + fsc_register_file(file_ptr, fs, eh); + if (qp_ext && !fsc_stricmp(qp_ext, "pk3") && (!qp_dir || !fsc_stricmp(qp_dir, "downloads"))) + { + fsc_load_pk3(STACKPTR(file->os_path_ptr), fs, file_ptr, eh, 0, 0); + file->f.flags |= FSC_FILEFLAG_LINKED_CONTENT; + } + } + + // Update stats + { + fsc_stats_t stats; + fsc_memset(&stats, 0, sizeof(stats)); + + stats.total_file_count = 1 + file->pk3_subfile_count; + + stats.cacheable_file_count = file->pk3_subfile_count; + if (file->shader_count || file->pk3_subfile_count) + ++stats.cacheable_file_count; + + stats.pk3_subfile_count = file->pk3_subfile_count; + + // By design, this field records only *valid* pk3s with a nonzero hash. + // Perhaps create another field that includes invalid pk3s? + if (file->pk3_hash) + stats.valid_pk3_count = 1; + + stats.shader_file_count = file->shader_file_count; + stats.shader_count = file->shader_count; + + fsc_merge_stats(&stats, &fs->active_stats); + if (unindexed_file) + fsc_merge_stats(&stats, &fs->total_stats); + if (new_file) + fsc_merge_stats(&stats, &fs->new_stats); + } +} + +static int fsc_app_extension(const char *name) +{ + // Returns 1 if name matches mac app bundle extension, 0 otherwise + int length = fsc_strlen(name); + if (length < 4) + return 0; + if (!fsc_stricmp(name + (length - 4), ".app")) + return 1; + return 0; +} + +void fsc_load_file_full_path(int source_dir_id, const void *os_path, const char *full_qpath, unsigned int os_timestamp, + unsigned int filesize, fsc_filesystem_t *fs, fsc_errorhandler_t *eh) +{ + char qp_buffer[FSC_MAX_QPATH]; + char qp_mod[FSC_MAX_MODDIR]; + const char *qpath_start = 0; + int file_in_pk3dir = 0; + char pk3dir_buffer[FSC_MAX_QPATH]; + const char *pk3dir_remainder = 0; + const char *qp_dir, *qp_name, *qp_ext; + + // Process mod directory prefix + if (!fsc_get_leading_directory(full_qpath, qp_mod, sizeof(qp_mod), &qpath_start)) + return; + if (!qpath_start) + return; + if (fsc_app_extension(qp_mod)) + return; // Don't index mac app bundles as mods + + // Process pk3dir prefix + if (fsc_get_leading_directory(qpath_start, pk3dir_buffer, sizeof(pk3dir_buffer), &pk3dir_remainder)) + { + if (pk3dir_remainder) + { + int length = fsc_strlen(pk3dir_buffer); + if (length >= 7 && !fsc_stricmp(pk3dir_buffer + length - 7, ".pk3dir")) + { + pk3dir_buffer[length - 7] = 0; + file_in_pk3dir = 1; + qpath_start = pk3dir_remainder; + } + } + } + + // Process qpath + if (!fsc_process_qpath(qpath_start, qp_buffer, &qp_dir, &qp_name, &qp_ext)) + return; + + // Load file + fsc_load_file(source_dir_id, os_path, qp_mod, file_in_pk3dir ? pk3dir_buffer : 0, qp_dir, qp_name, qp_ext, + os_timestamp, filesize, fs, eh); +} + +typedef struct { + int source_dir_id; + fsc_filesystem_t *fs; + fsc_errorhandler_t *eh; +} iterate_context_t; + +static void load_file_from_iteration(iterate_data_t *file_data, void *iterate_context) +{ + iterate_context_t *iterate_context_typed = (iterate_context_t *)iterate_context; + fsc_load_file_full_path(iterate_context_typed->source_dir_id, file_data->os_path, file_data->qpath_with_mod_dir, + file_data->os_timestamp, file_data->filesize, iterate_context_typed->fs, iterate_context_typed->eh); +} + +void fsc_filesystem_initialize(fsc_filesystem_t *fs) +{ + fsc_memset(fs, 0, sizeof(*fs)); + fsc_stack_initialize(&fs->general_stack); + fsc_hashtable_initialize(&fs->files, &fs->general_stack, 65536); + fsc_hashtable_initialize(&fs->string_repository, &fs->general_stack, 65536); + fsc_hashtable_initialize(&fs->directories, &fs->general_stack, 16384); + fsc_hashtable_initialize(&fs->shaders, &fs->general_stack, 65536); + fsc_hashtable_initialize(&fs->crosshairs, &fs->general_stack, 1); + fsc_hashtable_initialize(&fs->pk3_hash_lookup, &fs->general_stack, 4096); +} + +void fsc_filesystem_free(fsc_filesystem_t *fs) +{ + // Can be called on a nulled, freed, initialized, or in some cases partially initialized filesystem + fsc_stack_free(&fs->general_stack); + fsc_hashtable_free(&fs->files); + fsc_hashtable_free(&fs->string_repository); + fsc_hashtable_free(&fs->directories); + fsc_hashtable_free(&fs->shaders); + fsc_hashtable_free(&fs->crosshairs); + fsc_hashtable_free(&fs->pk3_hash_lookup); +} + +void fsc_filesystem_reset(fsc_filesystem_t *fs) +{ + ++fs->refresh_count; + fsc_memset(&fs->active_stats, 0, sizeof(fs->active_stats)); + fsc_memset(&fs->new_stats, 0, sizeof(fs->new_stats)); +} + +void fsc_load_directory(fsc_filesystem_t *fs, void *os_path, int source_dir_id, fsc_errorhandler_t *eh) +{ + iterate_context_t context; + context.source_dir_id = source_dir_id; + context.fs = fs; + context.eh = eh; + iterate_directory(os_path, load_file_from_iteration, &context); +} + +#endif // NEW_FILESYSTEM diff --git a/src/filesystem/fscore/fsc_md4.cpp b/src/filesystem/fscore/fsc_md4.cpp new file mode 100644 index 000000000..1114f2922 --- /dev/null +++ b/src/filesystem/fscore/fsc_md4.cpp @@ -0,0 +1,252 @@ +/* + mdfour.c + + An implementation of MD4 designed for use in the samba SMB + authentication protocol + + Copyright (C) 1997-1998 Andrew Tridgell + + This program is free software; you can redistribute it and/or + modify it under the terms of the GNU General Public License + as published by the Free Software Foundation; either version 2 + of the License, or (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. + + See the GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to: + + Free Software Foundation, Inc. + 59 Temple Place - Suite 330 + Boston, MA 02111-1307, USA + + $Id: mdfour.c,v 1.1 2002/08/23 22:03:27 abster Exp $ +*/ + +#ifdef NEW_FILESYSTEM +#include "fscore.h" +typedef unsigned int uint32_t; +typedef unsigned char byte; +#define Com_Memset fsc_memset +#define Com_Memcpy fsc_memcpy + +typedef struct mdfour { + uint32_t A, B, C, D; + uint32_t totalN; +} mdfour_t; + +/* NOTE: This code makes no attempt to be fast! + + It assumes that a int is at least 32 bits long +*/ + +static mdfour_t *m; + +#define F(X, Y, Z) (((X) & (Y)) | ((~(X)) & (Z))) +#define G(X, Y, Z) (((X) & (Y)) | ((X) & (Z)) | ((Y) & (Z))) +#define H(X, Y, Z) ((X) ^ (Y) ^ (Z)) +#define lshift(x, s) (((x) << (s)) | ((x) >> (32 - (s)))) + +#define ROUND1(a, b, c, d, k, s) a = lshift(a + F(b, c, d) + X[k], s) +#define ROUND2(a, b, c, d, k, s) a = lshift(a + G(b, c, d) + X[k] + 0x5A827999, s) +#define ROUND3(a, b, c, d, k, s) a = lshift(a + H(b, c, d) + X[k] + 0x6ED9EBA1, s) + +/* this applies md4 to 64 byte chunks */ +static void mdfour64(uint32_t *M) +{ + int j; + uint32_t AA, BB, CC, DD; + uint32_t X[16]; + uint32_t A, B, C, D; + + for (j = 0; j < 16; j++) + X[j] = M[j]; + + A = m->A; + B = m->B; + C = m->C; + D = m->D; + AA = A; + BB = B; + CC = C; + DD = D; + + ROUND1(A, B, C, D, 0, 3); + ROUND1(D, A, B, C, 1, 7); + ROUND1(C, D, A, B, 2, 11); + ROUND1(B, C, D, A, 3, 19); + ROUND1(A, B, C, D, 4, 3); + ROUND1(D, A, B, C, 5, 7); + ROUND1(C, D, A, B, 6, 11); + ROUND1(B, C, D, A, 7, 19); + ROUND1(A, B, C, D, 8, 3); + ROUND1(D, A, B, C, 9, 7); + ROUND1(C, D, A, B, 10, 11); + ROUND1(B, C, D, A, 11, 19); + ROUND1(A, B, C, D, 12, 3); + ROUND1(D, A, B, C, 13, 7); + ROUND1(C, D, A, B, 14, 11); + ROUND1(B, C, D, A, 15, 19); + + ROUND2(A, B, C, D, 0, 3); + ROUND2(D, A, B, C, 4, 5); + ROUND2(C, D, A, B, 8, 9); + ROUND2(B, C, D, A, 12, 13); + ROUND2(A, B, C, D, 1, 3); + ROUND2(D, A, B, C, 5, 5); + ROUND2(C, D, A, B, 9, 9); + ROUND2(B, C, D, A, 13, 13); + ROUND2(A, B, C, D, 2, 3); + ROUND2(D, A, B, C, 6, 5); + ROUND2(C, D, A, B, 10, 9); + ROUND2(B, C, D, A, 14, 13); + ROUND2(A, B, C, D, 3, 3); + ROUND2(D, A, B, C, 7, 5); + ROUND2(C, D, A, B, 11, 9); + ROUND2(B, C, D, A, 15, 13); + + ROUND3(A, B, C, D, 0, 3); + ROUND3(D, A, B, C, 8, 9); + ROUND3(C, D, A, B, 4, 11); + ROUND3(B, C, D, A, 12, 15); + ROUND3(A, B, C, D, 2, 3); + ROUND3(D, A, B, C, 10, 9); + ROUND3(C, D, A, B, 6, 11); + ROUND3(B, C, D, A, 14, 15); + ROUND3(A, B, C, D, 1, 3); + ROUND3(D, A, B, C, 9, 9); + ROUND3(C, D, A, B, 5, 11); + ROUND3(B, C, D, A, 13, 15); + ROUND3(A, B, C, D, 3, 3); + ROUND3(D, A, B, C, 11, 9); + ROUND3(C, D, A, B, 7, 11); + ROUND3(B, C, D, A, 15, 15); + + A += AA; + B += BB; + C += CC; + D += DD; + + for (j = 0; j < 16; j++) + X[j] = 0; + + m->A = A; + m->B = B; + m->C = C; + m->D = D; +} + +static void copy64(uint32_t *M, byte *in) +{ + int i; + + for (i = 0; i < 16; i++) + M[i] = (in[i * 4 + 3] << 24) | (in[i * 4 + 2] << 16) | (in[i * 4 + 1] << 8) | (in[i * 4 + 0] << 0); +} + +static void copy4(byte *out, uint32_t x) +{ + out[0] = x & 0xFF; + out[1] = (x >> 8) & 0xFF; + out[2] = (x >> 16) & 0xFF; + out[3] = (x >> 24) & 0xFF; +} + +static void mdfour_begin(struct mdfour *md) +{ + md->A = 0x67452301; + md->B = 0xefcdab89; + md->C = 0x98badcfe; + md->D = 0x10325476; + md->totalN = 0; +} + +static void mdfour_tail(byte *in, int n) +{ + byte buf[128]; + uint32_t M[16]; + uint32_t b; + + m->totalN += n; + + b = m->totalN * 8; + + Com_Memset(buf, 0, 128); + if (n) + Com_Memcpy(buf, in, n); + buf[n] = 0x80; + + if (n <= 55) + { + copy4(buf + 56, b); + copy64(M, buf); + mdfour64(M); + } + else + { + copy4(buf + 120, b); + copy64(M, buf); + mdfour64(M); + copy64(M, buf + 64); + mdfour64(M); + } +} + +static void mdfour_update(mdfour_t *md, byte *in, int n) +{ + uint32_t M[16]; + + m = md; + + if (n == 0) + mdfour_tail(in, n); + + while (n >= 64) + { + copy64(M, in); + mdfour64(M); + in += 64; + n -= 64; + m->totalN += 64; + } + + mdfour_tail(in, n); +} + +static void mdfour_result(mdfour_t *md, byte *out) +{ + m = md; + + copy4(out, m->A); + copy4(out + 4, m->B); + copy4(out + 8, m->C); + copy4(out + 12, m->D); +} + +static void mdfour(byte *out, byte *in, int n) +{ + mdfour_t md; + mdfour_begin(&md); + mdfour_update(&md, in, n); + mdfour_result(&md, out); +} + +//=================================================================== + +unsigned int fsc_block_checksum(const void *buffer, int length) +{ + int digest[4]; + unsigned val; + + mdfour((byte *)digest, (byte *)buffer, length); + + val = digest[0] ^ digest[1] ^ digest[2] ^ digest[3]; + + return val; +} + +#endif // NEW_FILESYSTEM diff --git a/src/filesystem/fscore/fsc_misc.cpp b/src/filesystem/fscore/fsc_misc.cpp new file mode 100644 index 000000000..96ff595a2 --- /dev/null +++ b/src/filesystem/fscore/fsc_misc.cpp @@ -0,0 +1,624 @@ +/* +=========================================================================== +Copyright (C) 2017 Noah Metzger (chomenor@gmail.com) + +This file is part of Quake III Arena source code. + +Quake III Arena source code is free software; you can redistribute it +and/or modify it under the terms of the GNU General Public License as +published by the Free Software Foundation; either version 2 of the License, +or (at your option) any later version. + +Quake III Arena source code is distributed in the hope that it will be +useful, but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with Quake III Arena source code; if not, write to the Free Software +Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA +=========================================================================== +*/ + +#ifdef NEW_FILESYSTEM +#include "fscore.h" + +#define STACKPTR_LCL(pointer) (FSC_STACK_RETRIEVE(stack, pointer, 0)) // non-null, local stack parameter + +/* ******************************************************************************** */ +// Misc +/* ******************************************************************************** */ + +unsigned int fsc_string_hash(const char *input1, const char *input2) +{ + // Only processes alphanumeric characters, so any symbol sanitizing routines don't change hash + unsigned int hash = 5381; + int c; + + if (input1) + while ((c = *(unsigned char *)input1++)) + { + if (c >= 'A' && c <= 'Z') + c += 'a' - 'A'; + if ((c >= 'a' && c <= 'z') || (c >= '0' && c <= '9')) + hash = ((hash << 5) + hash) + c; + } + if (input2) + while ((c = *(unsigned char *)input2++)) + { + if (c >= 'A' && c <= 'Z') + c += 'a' - 'A'; + if ((c >= 'a' && c <= 'z') || (c >= '0' && c <= '9')) + hash = ((hash << 5) + hash) + c; + } + + return hash; +} + +unsigned int fsc_fs_size_estimate(fsc_filesystem_t *fs) +{ + return fsc_stack_get_export_size(&fs->general_stack) + fsc_hashtable_get_export_size(&fs->string_repository) + + fsc_hashtable_get_export_size(&fs->files) + fsc_hashtable_get_export_size(&fs->directories) + + fsc_hashtable_get_export_size(&fs->shaders) + fsc_hashtable_get_export_size(&fs->crosshairs) + + fsc_hashtable_get_export_size(&fs->pk3_hash_lookup); +} + +/* ******************************************************************************** */ +// Data Stream +/* ******************************************************************************** */ + +int fsc_read_stream_data(fsc_stream_t *stream, void *output, unsigned int length) +{ + // Returns 1 on error, 0 on success. + FSC_ASSERT(stream); + FSC_ASSERT(output); + if (stream->position + length > stream->size || stream->position + length < stream->position) + return 1; + fsc_memcpy(output, stream->data + stream->position, length); + stream->position += length; + return 0; +} + +int fsc_write_stream_data(fsc_stream_t *stream, void *data, unsigned int length) +{ + // Returns 1 on error, 0 on success. + FSC_ASSERT(stream); + FSC_ASSERT(data); + if (stream->position + length > stream->size || stream->position + length < stream->position) + return 1; + fsc_memcpy(stream->data + stream->position, data, length); + stream->position += length; + return 0; +} + +void fsc_stream_append_string_substituted(fsc_stream_t *stream, const char *string, const char *substitution_table) +{ + // Writes string to stream using character substitution table. + // If stream runs out of space, output is truncated. + // Stream data will always be null terminated. + FSC_ASSERT(stream); + FSC_ASSERT(stream->size > 0); + if (!string) + string = ""; + while (*string) + { + if (stream->position >= stream->size - 1) + { + stream->overflowed = 1; + break; + } + if (substitution_table) + { + stream->data[stream->position++] = substitution_table[*(unsigned char *)(string++)]; + } + else + { + stream->data[stream->position++] = *(string++); + } + } + if (stream->position >= stream->size) + stream->position = stream->size - 1; + stream->data[stream->position] = 0; +} + +void fsc_stream_append_string(fsc_stream_t *stream, const char *string) +{ + // Writes string to stream. + // If stream runs out of space, output is truncated. + // Stream data will always be null terminated. + fsc_stream_append_string_substituted(stream, string, 0); +} + +/* ******************************************************************************** */ +// Standard Stack +/* ******************************************************************************** */ + +// This "stack" structure isn't really a stack in the technical sense, I just called +// it that while I was still figuring out how it would work and then didn't want +// to change the name everywhere +// Basically this is a memory structure that allocates memory but never frees anything +// without freeing the whole structure. It also uses its own pointer format so it +// can be written and read back from a file and keep using the same pointers. + +#define STACK_INITIAL_BUCKETS 16 +#define STACK_BUCKET_POSITION_BITS 20 // Determines size of each bucket + +#define STACK_BUCKET_ID_BITS (32 - STACK_BUCKET_POSITION_BITS) +#define STACK_MAX_BUCKETS (1 << STACK_BUCKET_ID_BITS) +#define STACK_BUCKET_SIZE (1 << STACK_BUCKET_POSITION_BITS) +#define STACK_BUCKET_DATA_SIZE (STACK_BUCKET_SIZE - sizeof(fsc_stack_bucket_t)) + +static void stack_add_bucket(fsc_stack_t *stack) +{ + stack->buckets_position++; + FSC_ASSERT(stack->buckets_position >= 0); + FSC_ASSERT(stack->buckets_position < STACK_MAX_BUCKETS); + + // Resize the bucket array if necessary + if (stack->buckets_position >= stack->buckets_size) + { + int new_size; + fsc_stack_bucket_t **new_allocation; + + // Try to double the buckets allocation size + new_size = stack->buckets_size * 2; + if (new_size > STACK_MAX_BUCKETS) + new_size = STACK_MAX_BUCKETS; + + new_allocation = (fsc_stack_bucket_t **)fsc_malloc(new_size * sizeof(fsc_stack_bucket_t *)); + fsc_memcpy(new_allocation, stack->buckets, (stack->buckets_position + 1) * sizeof(fsc_stack_bucket_t *)); + fsc_free(stack->buckets); + stack->buckets = new_allocation; + stack->buckets_size = new_size; + } + + // Stack allocations are assumed to be zeroed + // so it needs to be accounted for if you change the allocation method here + stack->buckets[stack->buckets_position] = (fsc_stack_bucket_t *)fsc_calloc(STACK_BUCKET_SIZE); + stack->buckets[stack->buckets_position]->position = 0; +} + +void fsc_stack_initialize(fsc_stack_t *stack) +{ + stack->buckets_position = -1; // stack_add_bucket will increment to 0 + stack->buckets_size = STACK_INITIAL_BUCKETS; + stack->buckets = (fsc_stack_bucket_t **)fsc_malloc(stack->buckets_size * sizeof(fsc_stack_bucket_t *)); + stack_add_bucket(stack); +} + +void *fsc_stack_retrieve( + const fsc_stack_t *stack, const fsc_stackptr_t pointer, int allow_null, const char *caller, const char *expression) +{ + // Converts stackptr for a given stack to actual pointer + if (pointer) + { + int bucket = pointer >> STACK_BUCKET_POSITION_BITS; + unsigned int offset = pointer & ((1 << STACK_BUCKET_POSITION_BITS) - 1); + + if (bucket < 0 || bucket > stack->buckets_position || offset < sizeof(fsc_stack_bucket_t) || + offset - sizeof(fsc_stack_bucket_t) >= stack->buckets[bucket]->position) + { + fsc_fatal_error_tagged("stackptr out of range", caller, expression); + } + + return (void *)((char *)stack->buckets[bucket] + offset); + } + else + { + if (!allow_null) + fsc_fatal_error_tagged("unexpected null stackptr", caller, expression); + return 0; + } +} + +fsc_stackptr_t fsc_stack_allocate(fsc_stack_t *stack, unsigned int size) +{ + // Returns 0 on error + fsc_stack_bucket_t *bucket = stack->buckets[stack->buckets_position]; + unsigned int aligned_position = (bucket->position + 3) & ~3; + char *output; + + // Add a new bucket if we are out of space + FSC_ASSERT(size < STACK_BUCKET_DATA_SIZE); + if (size > STACK_BUCKET_DATA_SIZE - aligned_position) + { + stack_add_bucket(stack); + bucket = stack->buckets[stack->buckets_position]; + aligned_position = (bucket->position + 3) & ~3; + } + + // Allocate the entry + output = (char *)bucket + sizeof(*bucket) + aligned_position; + bucket->position = aligned_position + size; + + return stack->buckets_position * STACK_BUCKET_SIZE + (unsigned int)(output - (char *)bucket); +} + +void fsc_stack_free(fsc_stack_t *stack) +{ + // Can be called on a nulled, freed, initialized, or in some cases partially initialized stack + int i; + if (stack->buckets) + { + for (i = 0; i <= stack->buckets_position; ++i) + { + if (stack->buckets[i]) + fsc_free(stack->buckets[i]); + } + fsc_free(stack->buckets); + stack->buckets = 0; + } +} + +unsigned int fsc_stack_get_export_size(fsc_stack_t *stack) +{ + unsigned int size = 4; // 4 bytes for bucket count field + int i; + + // Then add the actual length of each bucket + 4 bytes for position field + for (i = 0; i <= stack->buckets_position; ++i) + { + size += stack->buckets[i]->position + 4; + } + + return size; +} + +int fsc_stack_export(fsc_stack_t *stack, fsc_stream_t *stream) +{ + // Returns 1 on error, 0 on success + int i; + + // Write the number of buckets + if (fsc_write_stream_data(stream, &stack->buckets_position, 4)) + return 1; + + // Write each bucket (current position followed by data) + for (i = 0; i <= stack->buckets_position; ++i) + { + if (fsc_write_stream_data(stream, &stack->buckets[i]->position, 4)) + return 1; + if (fsc_write_stream_data( + stream, (char *)stack->buckets[i] + sizeof(fsc_stack_bucket_t), stack->buckets[i]->position)) + return 1; + } + + return 0; +} + +int fsc_stack_import(fsc_stack_t *stack, fsc_stream_t *stream) +{ + // Returns 1 on error, 0 on success + int i; + + // Read number of active buckets + if (fsc_read_stream_data(stream, &stack->buckets_position, 4)) + return 1; + if (stack->buckets_position < 0 || stack->buckets_position >= STACK_MAX_BUCKETS) + return 1; + + // Allocate bucket array + stack->buckets_size = stack->buckets_position + 1; + if (stack->buckets_size < STACK_INITIAL_BUCKETS) + stack->buckets_size = STACK_INITIAL_BUCKETS; + stack->buckets = (fsc_stack_bucket_t **)fsc_calloc(stack->buckets_size * sizeof(fsc_stack_bucket_t *)); + + // Read data for each bucket + for (i = 0; i <= stack->buckets_position; ++i) + { + stack->buckets[i] = (fsc_stack_bucket_t *)fsc_calloc(STACK_BUCKET_SIZE); + if (fsc_read_stream_data(stream, &stack->buckets[i]->position, 4)) + goto error; + if (stack->buckets[i]->position >= STACK_BUCKET_SIZE) + goto error; + if (fsc_read_stream_data( + stream, (char *)stack->buckets[i] + sizeof(fsc_stack_bucket_t), stack->buckets[i]->position)) + goto error; + } + return 0; + +error: + fsc_stack_free(stack); + return 1; +} + +/* ******************************************************************************** */ +// Standard Hash Table +/* ******************************************************************************** */ + +void fsc_hashtable_initialize(fsc_hashtable_t *ht, fsc_stack_t *stack, int bucket_count) +{ + if (bucket_count < 1) + bucket_count = 1; + if (bucket_count > FSC_HASHTABLE_MAX_BUCKETS) + bucket_count = FSC_HASHTABLE_MAX_BUCKETS; + ht->bucket_count = bucket_count; + ht->buckets = (fsc_stackptr_t *)fsc_calloc(sizeof(fsc_stackptr_t) * bucket_count); + ht->utilization = 0; + ht->stack = stack; +} + +void fsc_hashtable_open(fsc_hashtable_t *ht, unsigned int hash, fsc_hashtable_iterator_t *iterator) +{ + iterator->stack = ht->stack; + iterator->next_ptr = &ht->buckets[hash % ht->bucket_count]; +} + +fsc_stackptr_t fsc_hashtable_next(fsc_hashtable_iterator_t *iterator) +{ + fsc_stackptr_t current = *iterator->next_ptr; + if (current) + iterator->next_ptr = &(((fsc_hashtable_entry_t *)FSC_STACK_RETRIEVE(iterator->stack, current, 0))->next); + return current; +} + +void fsc_hashtable_insert(fsc_stackptr_t entry_ptr, unsigned int hash, fsc_hashtable_t *ht) +{ + // entry_ptr must be castable to fsc_hashtable_entry_t. + fsc_hashtable_entry_t *entry = (fsc_hashtable_entry_t *)FSC_STACK_RETRIEVE(ht->stack, entry_ptr, 0); + fsc_stackptr_t *bucket = &ht->buckets[hash % ht->bucket_count]; + entry->next = *bucket; + *bucket = entry_ptr; + ++ht->utilization; +} + +void fsc_hashtable_free(fsc_hashtable_t *ht) +{ + // Can be called on a nulled, freed, initialized, or in some cases partially initialized hashtable + if (ht->buckets) + fsc_free(ht->buckets); + ht->buckets = 0; +} + +unsigned int fsc_hashtable_get_export_size(fsc_hashtable_t *ht) { return 8 + ht->bucket_count * sizeof(*ht->buckets); } + +int fsc_hashtable_export(fsc_hashtable_t *ht, fsc_stream_t *stream) +{ + // Returns 1 on error, 0 on success. + if (fsc_write_stream_data(stream, &ht->bucket_count, 4)) + return 1; + if (fsc_write_stream_data(stream, &ht->utilization, 4)) + return 1; + if (fsc_write_stream_data(stream, ht->buckets, ht->bucket_count * sizeof(*ht->buckets))) + return 1; + return 0; +} + +int fsc_hashtable_import(fsc_hashtable_t *ht, fsc_stack_t *stack, fsc_stream_t *stream) +{ + // Returns 1 on error, 0 on success. + if (fsc_read_stream_data(stream, &ht->bucket_count, 4)) + return 1; + if (ht->bucket_count < 1 || ht->bucket_count > FSC_HASHTABLE_MAX_BUCKETS) + return 1; + if (fsc_read_stream_data(stream, &ht->utilization, 4)) + return 1; + ht->buckets = (fsc_stackptr_t *)fsc_malloc(ht->bucket_count * sizeof(*ht->buckets)); + if (fsc_read_stream_data(stream, ht->buckets, ht->bucket_count * sizeof(*ht->buckets))) + { + fsc_hashtable_free(ht); + return 1; + } + ht->stack = stack; + return 0; +} + +/* ******************************************************************************** */ +// Standard String Repository +/* ******************************************************************************** */ + +fsc_stackptr_t fsc_string_repository_getentry( + const char *input, int allocate, fsc_hashtable_t *string_repository, fsc_stack_t *stack) +{ + // Return type is stringrepository_entry_t + + unsigned int hash = fsc_string_hash(input, 0); + fsc_hashtable_iterator_t hti; + fsc_stackptr_t sre_ptr; + stringrepository_entry_t *sre; + + // Get entry + fsc_hashtable_open(string_repository, hash, &hti); + while ((sre_ptr = fsc_hashtable_next(&hti))) + { + sre = (stringrepository_entry_t *)STACKPTR_LCL(sre_ptr); + if (!fsc_strcmp((char *)sre + sizeof(*sre), input)) + break; + } + + if (!sre_ptr && allocate) + { + // Allocate new entry + int length = fsc_strlen(input) + 1; // Include null terminator + sre_ptr = fsc_stack_allocate(stack, sizeof(stringrepository_entry_t) + length); + sre = (stringrepository_entry_t *)STACKPTR_LCL(sre_ptr); + fsc_memcpy((char *)sre + sizeof(*sre), input, length); + fsc_hashtable_insert(sre_ptr, hash, string_repository); + } + + return sre_ptr; +} + +fsc_stackptr_t fsc_string_repository_getstring( + const char *input, int allocate, fsc_hashtable_t *string_repository, fsc_stack_t *stack) +{ + // Return type is char * + fsc_stackptr_t entry = fsc_string_repository_getentry(input, allocate, string_repository, stack); + if (entry) + return entry + sizeof(stringrepository_entry_t); + return 0; +} + +/* ******************************************************************************** */ +// Qpath Handling +/* ******************************************************************************** */ + +const char *fsc_get_qpath_conversion_table(void) +{ + // Used to sanitize qpaths + static char qpath_conversion_table[256]; + static int qpath_conversion_table_initialized = 0; + int i; + + if (qpath_conversion_table_initialized) + return qpath_conversion_table; + + // Default to underscore + for (i = 0; i < 256; ++i) + qpath_conversion_table[i] = '_'; + + // Valid characters + qpath_conversion_table[0] = 0; + qpath_conversion_table['/'] = '/'; + qpath_conversion_table['\\'] = '/'; + qpath_conversion_table['.'] = '.'; + qpath_conversion_table['-'] = '-'; + for (i = '0'; i <= '9'; ++i) + qpath_conversion_table[i] = i; + for (i = 'a'; i <= 'z'; ++i) + qpath_conversion_table[i] = i; + for (i = 'A'; i <= 'Z'; ++i) + qpath_conversion_table[i] = i; + + qpath_conversion_table_initialized = 1; + return qpath_conversion_table; +} + +int fsc_process_qpath(const char *input, char *buffer, const char **qp_dir, const char **qp_name, const char **qp_ext) +{ + // Breaks input path into directory, name, and extension sections. + // Buffer is used to store the separated path data and should be length FSC_MAX_QPATH. + // Output qp_dir and qp_ext may be null if no directory or extension is available. + // Input qp_ext may be null to disable extension processing. + // Returns number of chars written to buffer (NOT including final null terminator) + // Buffer can be same as input for in-place processing + int i; + int period_pos = 0; + int slash_pos = 0; + FSC_ASSERT(input); + FSC_ASSERT(buffer); + FSC_ASSERT(qp_dir); + FSC_ASSERT(qp_name); + + // Write buffer; get period_pos and slash_pos + for (i = 0; i < FSC_MAX_QPATH - 1; ++i) + { + buffer[i] = input[i]; + if (!buffer[i]) + break; + if (buffer[i] == '\\') + buffer[i] = '/'; + if (buffer[i] == '/') + { + slash_pos = i; + period_pos = 0; + } + if (buffer[i] == '.' && qp_ext) + period_pos = i; + } + buffer[i] = 0; + + // Break up output based on period and slash positions + if (slash_pos) + { + *qp_dir = buffer; + *qp_name = buffer + slash_pos + 1; + buffer[slash_pos] = 0; + } + else + { + *qp_dir = 0; + *qp_name = buffer; + } + + if (period_pos) + { + *qp_ext = buffer + period_pos + 1; + buffer[period_pos] = 0; + } + else + { + if (qp_ext) + *qp_ext = 0; + } + + return i; +} + +unsigned int fsc_get_leading_directory( + const char *input, char *buffer, unsigned int buffer_length, const char **remainder) +{ + // Writes leading directory (text before first slash) to buffer, truncating on overflow + // Writes pointer to remaining (post-slash) string to remainder, or null if not found + // Returns total number of chars in leading directory, without truncation, not counting null terminator. + // If (return value >= buffer_length) output was truncated. + unsigned int i; + unsigned int chars_written; + FSC_ASSERT(input); + FSC_ASSERT(buffer); + FSC_ASSERT(buffer_length > 0); + + // Start with null remainder + if (remainder) + *remainder = 0; + + // Write buffer and remainder (if slash encountered) + for (i = 0; input[i]; ++i) + { + if (input[i] == '/' || input[i] == '\\') + { + if (remainder) + *remainder = (char *)(input + i + 1); + break; + } + if (i < buffer_length) + buffer[i] = input[i]; + } + + chars_written = i < buffer_length ? i : buffer_length - 1; + buffer[chars_written] = 0; + return i; +} + +/* ******************************************************************************** */ +// Error Handling +/* ******************************************************************************** */ + +void fsc_report_error(fsc_errorhandler_t *errorhandler, int id, const char *msg, void *current_element) +{ + if (!errorhandler) + return; + errorhandler->handler(id, msg, current_element, errorhandler->context); +} + +static void (*fsc_registered_fatal_error_handler)(const char *msg) = 0; + +void fsc_register_fatal_error_handler(void (*handler)(const char *msg)) +{ + // Registers a handler function to call in the event fsc_fatal_error is invoked. + fsc_registered_fatal_error_handler = handler; +} + +void fsc_fatal_error(const char *msg) +{ + // Calls fatal error handler if registered. If not registered, or it returns, aborts the program. + if (fsc_registered_fatal_error_handler) + fsc_registered_fatal_error_handler(msg); + fsc_error_abort(msg); +} + +void fsc_fatal_error_tagged(const char *msg, const char *caller, const char *expression) +{ + // Calls fatal error handler with calling function and expression logging parameters + char buffer[1024]; + fsc_stream_t stream = {buffer, 0, sizeof(buffer), 0}; + fsc_stream_append_string(&stream, msg); + fsc_stream_append_string(&stream, " - function("); + fsc_stream_append_string(&stream, caller); + fsc_stream_append_string(&stream, ") expression("); + fsc_stream_append_string(&stream, expression); + fsc_stream_append_string(&stream, ")"); + fsc_fatal_error(buffer); +} + +#endif // NEW_FILESYSTEM diff --git a/src/filesystem/fscore/fsc_os.cpp b/src/filesystem/fscore/fsc_os.cpp new file mode 100644 index 000000000..40d62e0f2 --- /dev/null +++ b/src/filesystem/fscore/fsc_os.cpp @@ -0,0 +1,498 @@ +/* +=========================================================================== +Copyright (C) 2017 Noah Metzger (chomenor@gmail.com) + +This file is part of Quake III Arena source code. + +Quake III Arena source code is free software; you can redistribute it +and/or modify it under the terms of the GNU General Public License as +published by the Free Software Foundation; either version 2 of the License, +or (at your option) any later version. + +Quake III Arena source code is distributed in the hope that it will be +useful, but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with Quake III Arena source code; if not, write to the Free Software +Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA +=========================================================================== +*/ + +#ifdef NEW_FILESYSTEM +#include "fscore.h" + +/* ******************************************************************************** */ +// Headers / Definitions +/* ******************************************************************************** */ + +#ifdef _WIN32 +// Windows defines +#define _CRT_SECURE_NO_WARNINGS +#include +#ifdef UNICODE +#define WIN_UNICODE +#endif +#else +// Non-windows defines +#include +#include +#include +#include +#include +#include +#endif +// Common defines +#include + +/* ******************************************************************************** */ +// Misc Library Functions +/* ******************************************************************************** */ + +void fsc_error_abort(const char *msg) +{ + // Prints error message and aborts program + fprintf(stderr, "filesystem error: %s\n", msg); + exit(1); +} + +int fsc_rename_file(void *source_os_path, void *target_os_path) +{ + // Returns 1 on error, 0 on success + FSC_ASSERT(source_os_path && target_os_path); +#ifdef _WIN32 + if (!MoveFile((LPCTSTR)source_os_path, (LPCTSTR)target_os_path)) + return 1; +#else + if (rename((const char *)source_os_path, (const char *)target_os_path)) + return 1; +#endif + return 0; +} + +int fsc_delete_file(void *os_path) +{ + // Returns 1 on error, 0 on success + FSC_ASSERT(os_path); +#ifdef _WIN32 + // Can we just use remove instead? + if (!DeleteFile((LPCTSTR)os_path)) + return 1; +#else + if (!remove((const char *)os_path)) + return 1; +#endif + return 0; +} + +int fsc_mkdir(void *os_path) +{ + // Returns 1 on error, 0 on success or directory already exists + FSC_ASSERT(os_path); +#ifdef _WIN32 + if (CreateDirectory((LPCTSTR)os_path, 0)) + return 0; + return GetLastError() != ERROR_ALREADY_EXISTS; +#else + if (mkdir((const char *)os_path, 0750)) + return errno != EEXIST; +#endif + return 0; +} + +void *fsc_open_file(const void *os_path, const char *mode) +{ + FSC_ASSERT(os_path); + FSC_ASSERT(mode); + { +#ifdef WIN_UNICODE + int i; + wchar_t mode_wide[10]; + for (i = 0; i < 9; ++i) + { + mode_wide[i] = mode[i]; + if (!mode[i]) + break; + } + mode_wide[9] = 0; + return _wfopen((const wchar_t *)os_path, mode_wide); +#else + return fopen((const char *)os_path, mode); +#endif + } +} + +void fsc_fclose(void *fp) { fclose((FILE *)fp); } + +unsigned int fsc_fread(void *dest, int size, void *fp) +{ + FSC_ASSERT(dest); + return fread(dest, 1, size, (FILE *)fp); +} + +unsigned int fsc_fwrite(const void *src, int size, void *fp) +{ + FSC_ASSERT(src); + return fwrite(src, 1, size, (FILE *)fp); +} + +void fsc_fflush(void *fp) { fflush((FILE *)fp); } + +int fsc_fseek(void *fp, int offset, fsc_seek_type_t type) +{ + int os_type = SEEK_SET; + if (type == FSC_SEEK_CUR) + os_type = SEEK_CUR; + if (type == FSC_SEEK_END) + os_type = SEEK_END; + return fseek((FILE *)fp, offset, os_type); +} + +int fsc_fseek_set(void *fp, unsigned int offset) +{ + // Returns 1 on error, 0 otherwise + fsc_seek_type_t type = FSC_SEEK_SET; + do + { + unsigned int seek_amount = offset; + if (seek_amount > 2000000000) + seek_amount = 2000000000; + if (fsc_fseek(fp, (int)seek_amount, type)) + return 1; + offset -= seek_amount; + type = FSC_SEEK_CUR; + } while (offset); + return 0; +} + +unsigned int fsc_ftell(void *fp) +{ +// Returns 4294967295 on error or overflow +#ifdef _WIN32 + long long value = _ftelli64((FILE *)fp); +#else + off_t value = ftello((FILE *)fp); +#endif + if (value < 0 || value > 4294967295u) + return 4294967295u; + return (unsigned int)value; +} + +void fsc_memcpy(void *dst, const void *src, unsigned int size) +{ + FSC_ASSERT(dst && src); + memcpy(dst, src, size); +} + +int fsc_memcmp(const void *str1, const void *str2, unsigned int size) +{ + FSC_ASSERT(str1 && str2); + return memcmp(str1, str2, size); +} + +void fsc_memset(void *dst, int value, unsigned int size) +{ + FSC_ASSERT(dst); + memset(dst, value, size); +} + +void fsc_strncpy(char *dst, const char *src, unsigned int size) +{ + // Ensures null termination if size > 0 + FSC_ASSERT(dst && src); + strncpy(dst, src, size); + if (size) + dst[size - 1] = 0; +} + +void fsc_strncpy_lower(char *dst, const char *src, unsigned int size) +{ + FSC_ASSERT(dst && src); + if (size) + { + while (--size) + *(dst++) = tolower(*(src++)); + *dst = 0; + } +} + +int fsc_strcmp(const char *str1, const char *str2) +{ + FSC_ASSERT(str1 && str2); + return strcmp(str1, str2); +} + +int fsc_stricmp(const char *str1, const char *str2) +{ + FSC_ASSERT(str1 && str2); +#ifdef _WIN32 + return _stricmp(str1, str2); +#else + return strcasecmp(str1, str2); +#endif +} + +int fsc_strlen(const char *str) +{ + FSC_ASSERT(str); + return strlen(str); +} + +void *fsc_malloc(unsigned int size) +{ + void *result = malloc(size); + FSC_ASSERT(result); + return result; +} + +void *fsc_calloc(unsigned int size) +{ + void *result = calloc(size, 1); + FSC_ASSERT(result); + return result; +} + +void fsc_free(void *allocation) +{ + FSC_ASSERT(allocation); + free(allocation); +} + +/* ******************************************************************************** */ +// OS Path Handling +/* ******************************************************************************** */ + +#ifdef WIN_UNICODE +#define FSC_CHAR wchar_t +#define FSC_UCHAR unsigned short +#else +#define FSC_CHAR char +#define FSC_UCHAR unsigned char +#endif + +void *fsc_string_to_os_path(const char *path) +{ + // Converts UTF-8 string to OS path format + // WARNING: Result must be freed by caller using fsc_free!!! + FSC_ASSERT(path); + { + FSC_CHAR *buffer; +#ifdef WIN_UNICODE + int length = MultiByteToWideChar(CP_UTF8, 0, path, -1, 0, 0); + FSC_ASSERT(length); + buffer = (FSC_CHAR *)fsc_malloc(length * sizeof(wchar_t)); + MultiByteToWideChar(CP_UTF8, 0, path, -1, buffer, length); +#else + // Consider stripping non-ASCII chars for non-unicode Windows builds + int length = strlen(path); + buffer = (char *)fsc_malloc(length + 1); + fsc_memcpy(buffer, path, length + 1); +#endif + return buffer; + } +} + +char *fsc_os_path_to_string(const void *os_path) +{ + // Converts OS path format to UTF-8 string + // WARNING: Result must be freed by caller using fsc_free!!! + FSC_ASSERT(os_path); + { + char *buffer; +#ifdef WIN_UNICODE + int length = WideCharToMultiByte(CP_UTF8, 0, (LPCWSTR)os_path, -1, 0, 0, 0, 0); + FSC_ASSERT(length); + buffer = (char *)fsc_malloc(length); + WideCharToMultiByte(CP_UTF8, 0, (LPCWSTR)os_path, -1, buffer, length, 0, 0); +#else + int length = strlen((const char *)os_path); + buffer = (char *)fsc_malloc(length + 1); + fsc_memcpy(buffer, os_path, length + 1); +#endif + return buffer; + } +} + +int fsc_os_path_size(const void *os_path) +{ + // Length in bytes + FSC_ASSERT(os_path); +#ifdef WIN_UNICODE + return 2 * wcslen((const wchar_t *)os_path) + 2; +#else + return strlen((const char *)os_path) + 1; +#endif +} + +int fsc_compare_os_path(const void *path1, const void *path2) +{ + // Returns 0 if paths are equal + FSC_ASSERT(path1 && path2); +#ifdef WIN_UNICODE + return wcscmp((const wchar_t *)path1, (const wchar_t *)path2); +#else + return strcmp((const char *)path1, (const char *)path2); +#endif +} + +/* ******************************************************************************** */ +// Directory Iteration +/* ******************************************************************************** */ + +#define SEARCH_PATH_LIMIT 260 + +typedef struct { + FSC_CHAR path[SEARCH_PATH_LIMIT]; + int path_position; + int base_length; + + void (*operation)(iterate_data_t *file_data, void *iterate_context); + char qpath_buffer[FSC_MAX_QPATH]; + iterate_data_t file_data; + void *iterate_context; +} iterate_work_t; + +// Returns 0 on overflow, 1 on success +static int iterate_append_path(iterate_work_t *iw, const FSC_CHAR *path) +{ + while (iw->path_position < SEARCH_PATH_LIMIT) + { + iw->path[iw->path_position] = *path; + if (!*(path++)) + return 1; + ++iw->path_position; + } + + iw->path_position = SEARCH_PATH_LIMIT - 1; + iw->path[iw->path_position] = 0; + return 0; +} + +static void iterate_set_position(iterate_work_t *iw, int position) +{ + FSC_ASSERT(position >= 0 && position < SEARCH_PATH_LIMIT); + iw->path_position = position; + iw->path[position] = 0; +} + +#ifdef _WIN32 +static void iterate_directory2(iterate_work_t *iw, int junction_allowed) +{ + int old_position = iw->path_position; + WIN32_FIND_DATA FindFileData; + HANDLE hFind; + + if (!iterate_append_path(iw, TEXT("\\*"))) + return; + hFind = FindFirstFile(iw->path, &FindFileData); + // hFind = FindFirstFileEx(search, FindExInfoBasic, &FindFileData, FindExSearchNameMatch, 0, + // FIND_FIRST_EX_LARGE_FETCH); + if (hFind == INVALID_HANDLE_VALUE) + return; + + do + { + // Prepare path + iterate_set_position(iw, old_position); + iterate_append_path(iw, TEXT("\\")); + if (!iterate_append_path(iw, FindFileData.cFileName)) + continue; + + if (FindFileData.dwFileAttributes & FILE_ATTRIBUTE_DIRECTORY) + { + // Have directory - check validity + if ((FindFileData.dwFileAttributes & FILE_ATTRIBUTE_REPARSE_POINT) && !junction_allowed) + continue; + if (FindFileData.cFileName[0] == '.' && + (!FindFileData.cFileName[1] || (FindFileData.cFileName[1] == '.' && !FindFileData.cFileName[2]))) + continue; + + iterate_directory2(iw, junction_allowed); + } + else + { + // Skip files greater than 4GB; they are not currently supported + if (FindFileData.nFileSizeHigh) + continue; + + // Add filename to path, to generate complete os_path + iw->file_data.os_path = iw->path; + iw->file_data.qpath_with_mod_dir = fsc_os_path_to_string(iw->path + iw->base_length + 1); + iw->file_data.os_timestamp = FindFileData.ftLastWriteTime.dwLowDateTime; + iw->file_data.filesize = FindFileData.nFileSizeLow; + iw->operation(&iw->file_data, iw->iterate_context); + fsc_free(iw->file_data.qpath_with_mod_dir); + } + } while (FindNextFile(hFind, &FindFileData)); + + FindClose(hFind); +} +#else +static void iterate_directory2(iterate_work_t *iw, int junction_allowed) +{ + int old_position = iw->path_position; + DIR *dir = opendir(iw->path); + if (!dir) + return; + + while (1) + { + struct dirent *entry; + struct stat st; + + // Get next entry + entry = readdir(dir); + if (!entry) + break; + + // Get path and stat + iterate_set_position(iw, old_position); + iterate_append_path(iw, "/"); + if (!iterate_append_path(iw, entry->d_name)) + continue; + if (stat(iw->path, &st) == -1) + continue; + + if (entry->d_type & DT_DIR) + { + // Have directory - check validity + if (entry->d_name[0] == '.' && (!entry->d_name[1] || (entry->d_name[1] == '.' && !entry->d_name[2]))) + continue; + + iterate_directory2(iw, junction_allowed); + } + else + { + // Skip files greater than 4GB; they are not currently supported + if (st.st_size > 4294967295u) + continue; + + iw->file_data.os_path = iw->path; + iw->file_data.qpath_with_mod_dir = fsc_os_path_to_string(iw->path + iw->base_length + 1); + iw->file_data.os_timestamp = (unsigned int)st.st_mtime; + iw->file_data.filesize = (unsigned int)st.st_size; + iw->operation(&iw->file_data, iw->iterate_context); + fsc_free(iw->file_data.qpath_with_mod_dir); + } + } + + closedir(dir); +} +#endif + +void iterate_directory( + void *search_os_path, void(operation)(iterate_data_t *file_data, void *iterate_context), void *iterate_context) +{ + iterate_work_t iw; + iw.path_position = 0; + iw.operation = operation; + iw.iterate_context = iterate_context; + + iterate_append_path(&iw, (const FSC_CHAR *)search_os_path); + iw.base_length = iw.path_position; + + iterate_directory2(&iw, 0); +} + +#endif // NEW_FILESYSTEM diff --git a/src/filesystem/fscore/fsc_pk3.cpp b/src/filesystem/fscore/fsc_pk3.cpp new file mode 100644 index 000000000..6324f12c5 --- /dev/null +++ b/src/filesystem/fscore/fsc_pk3.cpp @@ -0,0 +1,567 @@ +/* +=========================================================================== +Copyright (C) 2017 Noah Metzger (chomenor@gmail.com) + +This file is part of Quake III Arena source code. + +Quake III Arena source code is free software; you can redistribute it +and/or modify it under the terms of the GNU General Public License as +published by the Free Software Foundation; either version 2 of the License, +or (at your option) any later version. + +Quake III Arena source code is distributed in the hope that it will be +useful, but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with Quake III Arena source code; if not, write to the Free Software +Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA +=========================================================================== +*/ + +#ifdef NEW_FILESYSTEM +#include "fscore.h" + +#ifdef USE_LOCAL_HEADERS +#include "../../zlib/zlib.h" +#else +#include +#endif + +#define STACKPTR(pointer) (FSC_STACK_RETRIEVE(&fs->general_stack, pointer, 0)) // non-null +#define STACKPTRN(pointer) (FSC_STACK_RETRIEVE(&fs->general_stack, pointer, 1)) // null allowed +#define STACKPTR_LCL(pointer) (FSC_STACK_RETRIEVE(stack, pointer, 0)) // non null, local stack parameter + +// Somewhat arbitrary limit to avoid overflow issues +#define FSC_MAX_PK3_SIZE 4240000000u + +/* ******************************************************************************** */ +// PK3 File Processing +/* ******************************************************************************** */ + +typedef struct { + char *data; + int cd_length; + unsigned int zip_offset; + int entry_count; +} central_directory_t; + +static int fsc_endian_check(void) +{ + // Returns 1 on little endian system, 0 on big endian system + // WARNING: Currently untested on big endian system + static volatile int test = 1; + if (*(char *)&test) + return 1; + return 0; +} + +static unsigned int fsc_endian_convert_int(unsigned int value) +{ + if (fsc_endian_check()) + return value; + value = ((value << 8) & 0xFF00FF00) | ((value >> 8) & 0xFF00FF); + return (value << 16) | (value >> 16); +} + +static unsigned short fsc_endian_convert_short(unsigned short value) +{ + if (fsc_endian_check()) + return value; + return (value << 8) | (value >> 8); +} + +static int get_pk3_central_directory_fp(void *fp, unsigned int file_length, central_directory_t *output) +{ + // Returns 1 on error, 0 on success + char buffer[66000]; + int buffer_read_size = 0; // Offset from end of file/buffer of data that has been successfully read to buffer + int eocd_position = 0; // Offset from end of file/buffer of the EOCD record + unsigned int cd_position; // Offset from beginning of file of central directory + + // End Of Central Directory Record (EOCD) can start anywhere in the last 65KB or so of the zip file, + // determined by the presence of a magic number. For performance purposes first scan the last 4KB of the file, + // and if the magic number isn't found do a second pass for the whole 65KB. + int pass; + for (pass = 0; pass < 2; ++pass) + { + // Get buffer_read_target, which is the offset from end of file/buffer that we are tring to read to buffer + int i; + int buffer_read_target = pass == 0 ? 4096 : sizeof(buffer); + if ((unsigned int)buffer_read_target > file_length) + buffer_read_target = file_length; + if (buffer_read_target <= buffer_read_size) + return 1; + + // Read the data + fsc_fseek_set(fp, file_length - buffer_read_target); + fsc_fread(buffer + sizeof(buffer) - buffer_read_target, buffer_read_target - buffer_read_size, fp); + + // Search for magic number + for (i = 22; i < buffer_read_target; ++i) + { // EOCD cannot start less than 22 bytes from end of file, because it is 22 bytes long + char *string = buffer + sizeof(buffer) - i; + if (string[0] == 0x50 && string[1] == 0x4b && string[2] == 0x05 && string[3] == 0x06) + { + eocd_position = i; + break; + } + } + + buffer_read_size = buffer_read_target; + if (eocd_position) + break; + } + + if (!eocd_position) + return 1; + +#define EOCD_SHORT(offset) \ + fsc_endian_convert_short(*(unsigned short *)(buffer + sizeof(buffer) - eocd_position + offset)) +#define EOCD_INT(offset) fsc_endian_convert_int(*(unsigned int *)(buffer + sizeof(buffer) - eocd_position + offset)) + + output->entry_count = EOCD_SHORT(8); + output->cd_length = EOCD_INT(12); + + // No reason for central directory to be over 100 MB + if (output->cd_length < 0 || output->cd_length > 100 << 20) + return 1; + + // Must be space for central directory between the beginning of the file and the EOCD start + if ((unsigned int)output->cd_length > file_length - eocd_position) + return 1; + + // EOCD sanity checks that were in the original code: something to do with ensuring this is not a spanned archive + if (EOCD_INT(4) != 0) + return 1; // "this file's disk number" and "disk number containing central directory" should both be 0 + if (EOCD_SHORT(8) != EOCD_SHORT(8)) + return 1; // "cd entries on this disk" and "cd entries total" should be equal + + // Determine real central directory position + cd_position = file_length - eocd_position - output->cd_length; + + // Determine zip offset from error in reported cd position - all file offsets need to be adjusted by this value + { + unsigned int cd_position_reported = EOCD_INT(16); + if (cd_position_reported > cd_position) + return 1; // cd_position is already the maximum valid position + output->zip_offset = cd_position - cd_position_reported; + } + + output->data = (char *)fsc_malloc(output->cd_length); + + { + unsigned int buffer_file_position = + file_length - buffer_read_size; // Position in file where buffered data begins + unsigned int unbuffered_read_length = 0; // Amount of non-buffered data read + if (cd_position < buffer_file_position) + { + // Since central directory starts before buffer, read unbuffered part of central directory into output + unbuffered_read_length = buffer_file_position - cd_position; + if (unbuffered_read_length > (unsigned int)output->cd_length) + unbuffered_read_length = output->cd_length; + fsc_fseek_set(fp, cd_position); + fsc_fread(output->data, unbuffered_read_length, fp); + } + if (unbuffered_read_length < (unsigned int)output->cd_length) + { + // Read remaining data from buffer into output + char *buffer_data = buffer + sizeof(buffer) - buffer_read_size; + if (cd_position > buffer_file_position) + buffer_data += cd_position - buffer_file_position; + fsc_memcpy(output->data + unbuffered_read_length, buffer_data, output->cd_length - unbuffered_read_length); + } + } + + return 0; +} + +static int get_pk3_central_directory_path( + void *os_path, central_directory_t *output, fsc_file_direct_t *source_file, fsc_errorhandler_t *eh) +{ + // Returns 1 on error, 0 on success + void *fp = 0; + unsigned int length; + + // Open file + fp = fsc_open_file(os_path, "rb"); + if (!fp) + { + fsc_report_error(eh, FSC_ERROR_PK3FILE, "error opening pk3", source_file); + return 1; + } + + // Get size + fsc_fseek(fp, 0, FSC_SEEK_END); + length = fsc_ftell(fp); + if (!length) + { + fsc_fclose(fp); + fsc_report_error(eh, FSC_ERROR_PK3FILE, "zero size pk3", source_file); + return 1; + } + if (length > FSC_MAX_PK3_SIZE) + { + fsc_fclose(fp); + fsc_report_error(eh, FSC_ERROR_PK3FILE, "excessively large pk3", source_file); + return 1; + } + + // Get central directory + if (get_pk3_central_directory_fp(fp, length, output)) + { + fsc_fclose(fp); + fsc_report_error(eh, FSC_ERROR_PK3FILE, "error retrieving pk3 central directory", source_file); + return 1; + } + fsc_fclose(fp); + + return 0; +} + +void register_pk3_hash_lookup_entry(fsc_stackptr_t pk3_file_ptr, fsc_hashtable_t *pk3_hash_lookup, fsc_stack_t *stack) +{ + fsc_file_direct_t *pk3_file = (fsc_file_direct_t *)STACKPTR_LCL(pk3_file_ptr); + fsc_stackptr_t hash_map_entry_ptr = fsc_stack_allocate(stack, sizeof(fsc_pk3_hash_map_entry_t)); + fsc_pk3_hash_map_entry_t *hash_map_entry = (fsc_pk3_hash_map_entry_t *)STACKPTR_LCL(hash_map_entry_ptr); + hash_map_entry->pk3 = pk3_file_ptr; + fsc_hashtable_insert(hash_map_entry_ptr, pk3_file->pk3_hash, pk3_hash_lookup); +} + +static void register_file_from_pk3(fsc_filesystem_t *fs, char *filename, int filename_length, + fsc_stackptr_t sourcefile_ptr, unsigned int header_position, unsigned int compressed_size, + unsigned int uncompressed_size, short compression_method, fsc_errorhandler_t *eh) +{ + fsc_file_direct_t *sourcefile = (fsc_file_direct_t *)STACKPTR(sourcefile_ptr); + fsc_stackptr_t file_ptr = fsc_stack_allocate(&fs->general_stack, sizeof(fsc_file_frompk3_t)); + fsc_file_frompk3_t *file = (fsc_file_frompk3_t *)STACKPTR(file_ptr); + + char buffer[FSC_MAX_QPATH]; + const char *qp_dir, *qp_name, *qp_ext; + + // Copy filename into null-terminated buffer for process_qpath + // Also convert to lowercase to match behavior of old filesystem + if (filename_length >= FSC_MAX_QPATH) + filename_length = FSC_MAX_QPATH - 1; + fsc_strncpy_lower(buffer, filename, filename_length + 1); + + // Call process_qpath + fsc_process_qpath(buffer, buffer, &qp_dir, &qp_name, &qp_ext); + + // Write qpaths to file structure + file->f.qp_dir_ptr = + qp_dir ? fsc_string_repository_getstring(qp_dir, 1, &fs->string_repository, &fs->general_stack) : 0; + file->f.qp_name_ptr = fsc_string_repository_getstring(qp_name, 1, &fs->string_repository, &fs->general_stack); + file->f.qp_ext_ptr = + qp_ext ? fsc_string_repository_getstring(qp_ext, 1, &fs->string_repository, &fs->general_stack) : 0; + + // Load the rest of the fields + file->f.sourcetype = FSC_SOURCETYPE_PK3; + file->source_pk3 = sourcefile_ptr; + file->header_position = header_position; + file->compressed_size = compressed_size; + file->compression_method = compression_method; + file->f.filesize = uncompressed_size; + + // Register file and load contents + fsc_register_file(file_ptr, fs, eh); + ++sourcefile->pk3_subfile_count; +} + +void fsc_load_pk3(void *os_path, fsc_filesystem_t *fs, fsc_stackptr_t sourcefile_ptr, fsc_errorhandler_t *eh, + void (*receive_hash_data)(void *context, char *data, int size), void *receive_hash_data_context) +{ + // In normal usage, this is used to load pk3 files into the index. It can also be called with + // receive_hash_data set to generate pk3 hash data without indexing anything. + fsc_file_direct_t *sourcefile = (fsc_file_direct_t *)STACKPTRN(sourcefile_ptr); + central_directory_t cd; + int entry_position = 0; // Position of current entry relative to central directory data + int entry_counter = 0; // Number of current entry + + int filename_length; + int entry_length; + unsigned int uncompressed_size; + unsigned int compressed_size; + unsigned int header_position; + + int *crcs_for_hash; + int crcs_for_hash_buffer[1024]; + int crcs_for_hash_count = 0; + + if (!receive_hash_data) + FSC_ASSERT(sourcefile_ptr); + + // Load central directory + if (get_pk3_central_directory_path(os_path, &cd, sourcefile, eh)) + return; + + // Try to use the stack buffer, but if it's not big enough resort to malloc + if (cd.entry_count > sizeof(crcs_for_hash_buffer) / sizeof(*crcs_for_hash_buffer)) + { + crcs_for_hash = (int *)fsc_malloc((cd.entry_count + 1) * 4); + } + else + crcs_for_hash = crcs_for_hash_buffer; + + // Process each file + while (1) + { + // Make sure there is enough space to read the entry (minimum 47 bytes if filename is 1 byte) + if (entry_position + 47 > cd.cd_length) + { + fsc_report_error(eh, FSC_ERROR_PK3FILE, "invalid file cd entry position", sourcefile); + goto freemem; + } + + // Verify magic number + if (cd.data[entry_position] != 0x50 || cd.data[entry_position + 1] != 0x4b || + cd.data[entry_position + 2] != 0x01 || cd.data[entry_position + 3] != 0x02) + { + fsc_report_error(eh, FSC_ERROR_PK3FILE, "file cd entry does not have correct signature", sourcefile); + goto freemem; + } + +#define CD_ENTRY_SHORT(offset) fsc_endian_convert_short(*(unsigned short *)(cd.data + entry_position + offset)) +#define CD_ENTRY_INT(offset) fsc_endian_convert_int(*(unsigned int *)(cd.data + entry_position + offset)) +#define CD_ENTRY_INT_LE(offset) (*(unsigned int *)(cd.data + entry_position + offset)) + + // Get filename_length and entry_length + filename_length = (int)CD_ENTRY_SHORT(28); + { + int extrafield_length = (int)CD_ENTRY_SHORT(30); + int comment_length = (int)CD_ENTRY_SHORT(32); + entry_length = 46 + filename_length + extrafield_length + comment_length; + if (entry_position + entry_length > cd.cd_length) + { + fsc_report_error(eh, FSC_ERROR_PK3FILE, "invalid file cd entry position 2", sourcefile); + goto freemem; + } + } + + // Get compressed_size and uncompressed_size + compressed_size = CD_ENTRY_INT(20); + uncompressed_size = CD_ENTRY_INT(24); + + // Get local header_position (which is indicated by CD header, but needs to be modified by zip offset) + header_position = CD_ENTRY_INT(42) + cd.zip_offset; + + // Sanity checks + if (header_position + compressed_size < header_position) + { + fsc_report_error(eh, FSC_ERROR_PK3FILE, "invalid file local entry position 1", sourcefile); + goto freemem; + } + if (header_position + compressed_size > FSC_MAX_PK3_SIZE) + { + fsc_report_error(eh, FSC_ERROR_PK3FILE, "invalid file local entry position 2", sourcefile); + goto freemem; + } + + if (uncompressed_size) + { + crcs_for_hash[crcs_for_hash_count++] = CD_ENTRY_INT_LE(16); + } + + if (!(void *)receive_hash_data && + !(!uncompressed_size && *(cd.data + entry_position + 46 + filename_length - 1) == '/')) + { + // Not in hash mode and not a directory entry - load the file + register_file_from_pk3(fs, cd.data + entry_position + 46, filename_length, sourcefile_ptr, header_position, + compressed_size, CD_ENTRY_INT(24), CD_ENTRY_SHORT(10), eh); + } + + ++entry_counter; + entry_position += entry_length; + if (entry_counter >= cd.entry_count) + break; + } // Abort if that was the last file + + if ((void *)receive_hash_data) + { + receive_hash_data(receive_hash_data_context, (char *)crcs_for_hash, crcs_for_hash_count * 4); + goto freemem; + } + + sourcefile->pk3_hash = fsc_block_checksum(crcs_for_hash, crcs_for_hash_count * 4); + + // Add the pk3 to the hash lookup table + register_pk3_hash_lookup_entry(sourcefile_ptr, &fs->pk3_hash_lookup, &fs->general_stack); + +freemem: + fsc_free(cd.data); + if (crcs_for_hash != crcs_for_hash_buffer) + fsc_free(crcs_for_hash); +} + +/* ******************************************************************************** */ +// PK3 Extraction Support +/* ******************************************************************************** */ + +typedef struct { + void *input_handle; + int compression_method; + unsigned int input_remaining; // Remaining to be read from input handle + + // For zlib streams only + unsigned int input_buffer_size; + char *input_buffer; + z_stream zlib_stream; +} pk3_handle_t; + +static int fsc_pk3_handle_open2(pk3_handle_t *handle, const fsc_file_frompk3_t *file, int input_buffer_size, + const fsc_filesystem_t *fs, fsc_errorhandler_t *eh) +{ + // Returns 1 on error, 0 otherwise + const fsc_file_direct_t *source_pk3 = (const fsc_file_direct_t *)STACKPTR(file->source_pk3); + char localheader[30]; + unsigned int data_position; + + // Open the file + handle->input_handle = fsc_open_file(STACKPTR(source_pk3->os_path_ptr), "rb"); + if (!handle->input_handle) + { + fsc_report_error(eh, FSC_ERROR_EXTRACT, "pk3_handle_open - failed to open pk3 file", 0); + return 1; + } + + // Read the local header to get data position + fsc_fseek_set(handle->input_handle, file->header_position); + if (fsc_fread(localheader, 30, handle->input_handle) != 30) + { + fsc_report_error(eh, FSC_ERROR_EXTRACT, "pk3_handle_open - failed to read local header", 0); + return 1; + } + if (localheader[0] != 0x50 || localheader[1] != 0x4b || localheader[2] != 0x03 || localheader[3] != 0x04) + { + fsc_report_error(eh, FSC_ERROR_EXTRACT, "pk3_handle_open - incorrect signature in local header", 0); + return 1; + } +#define LH_SHORT(offset) fsc_endian_convert_short(*(unsigned short *)(localheader + offset)) + data_position = file->header_position + LH_SHORT(26) + LH_SHORT(28) + 30; + + // Seek to data start position + fsc_fseek_set(handle->input_handle, data_position); + + // Configure the handle + handle->input_remaining = file->compressed_size; + if (file->compression_method == 8) + { + if (inflateInit2(&handle->zlib_stream, -MAX_WBITS) != Z_OK) + { + fsc_report_error(eh, FSC_ERROR_EXTRACT, "pk3_handle_open - zlib inflateInit failed", 0); + return 1; + } + + handle->compression_method = 8; + handle->input_buffer_size = input_buffer_size; + handle->input_buffer = (char *)fsc_malloc(input_buffer_size); + } + else if (file->compression_method != 0) + { + fsc_report_error(eh, FSC_ERROR_EXTRACT, "pk3_handle_open - unknown compression method", 0); + return 1; + } + + return 0; +} + +void *fsc_pk3_handle_open( + const fsc_file_frompk3_t *file, int input_buffer_size, const fsc_filesystem_t *fs, fsc_errorhandler_t *eh) +{ + // Returns handle on success, null on error + pk3_handle_t *handle = (pk3_handle_t *)fsc_calloc(sizeof(*handle)); + if (fsc_pk3_handle_open2(handle, file, input_buffer_size, fs, eh)) + { + if (handle->input_handle) + fsc_fclose(handle->input_handle); + fsc_free(handle); + return 0; + } + return (void *)handle; +} + +void fsc_pk3_handle_close(void *handle) +{ + pk3_handle_t *Handle = (pk3_handle_t *)handle; + if (Handle->input_handle) + fsc_fclose(Handle->input_handle); + if (Handle->compression_method == 8) + { + fsc_free(Handle->input_buffer); + inflateEnd(&Handle->zlib_stream); + } + fsc_free(handle); +} + +unsigned int fsc_pk3_handle_read(void *handle, char *buffer, unsigned int length) +{ + // Returns number of bytes read + pk3_handle_t *Handle = (pk3_handle_t *)handle; + if (Handle->compression_method == 8) + { + Handle->zlib_stream.next_out = (Bytef *)buffer; + Handle->zlib_stream.avail_out = length; + + while (Handle->zlib_stream.avail_out) + { + if (!Handle->zlib_stream.avail_in) + { + // Load new batch of data into input + unsigned int feed_amount = Handle->input_remaining; + if (feed_amount > Handle->input_buffer_size) + feed_amount = Handle->input_buffer_size; + if (!feed_amount) + break; // Ran out of input + if (fsc_fread(Handle->input_buffer, (int)feed_amount, Handle->input_handle) != feed_amount) + break; + Handle->zlib_stream.avail_in += feed_amount; + Handle->input_remaining -= feed_amount; + Handle->zlib_stream.next_in = (Bytef *)Handle->input_buffer; + } + + if (inflate(&Handle->zlib_stream, Z_SYNC_FLUSH) != Z_OK) + break; + } + + return length - Handle->zlib_stream.avail_out; + } + else + { + return fsc_fread(buffer, length, Handle->input_handle); + } +} + +/* ******************************************************************************** */ +// PK3 Sourcetype Operations +/* ******************************************************************************** */ + +static int fsc_pk3_is_file_active(const fsc_file_t *file, const fsc_filesystem_t *fs) +{ + return fsc_get_base_file(file, fs)->refresh_count == fs->refresh_count; +} + +static const char *fsc_pk3_get_mod_dir(const fsc_file_t *file, const fsc_filesystem_t *fs) +{ + return (const char *)STACKPTR(fsc_get_base_file(file, fs)->qp_mod_ptr); +} + +static int fsc_pk3_extract_data( + const fsc_file_t *file, char *buffer, const fsc_filesystem_t *fs, fsc_errorhandler_t *eh) +{ + // Returns 0 on success, 1 on failure + int result = 0; + const fsc_file_frompk3_t *File = (fsc_file_frompk3_t *)file; + void *handle = fsc_pk3_handle_open(File, File->compressed_size, fs, eh); + if (!handle) + return 1; + if (fsc_pk3_handle_read(handle, buffer, file->filesize) != file->filesize) + result = 1; + fsc_pk3_handle_close(handle); + return result; +} + +fsc_sourcetype_t pk3_sourcetype = { + FSC_SOURCETYPE_PK3, fsc_pk3_is_file_active, fsc_pk3_get_mod_dir, fsc_pk3_extract_data}; + +#endif // NEW_FILESYSTEM diff --git a/src/filesystem/fscore/fsc_sha256.cpp b/src/filesystem/fscore/fsc_sha256.cpp new file mode 100644 index 000000000..96d46daa2 --- /dev/null +++ b/src/filesystem/fscore/fsc_sha256.cpp @@ -0,0 +1,381 @@ +/* Crypto/Sha256.c -- SHA-256 Hash +2015-11-14 : Igor Pavlov : Public domain +This code is based on public domain code from Wei Dai's Crypto++ library. */ +// From 7-zip; modified for ioquake3 + +#ifdef NEW_FILESYSTEM + +/* ******************************************************************************** */ +// Header substitutes +/* ******************************************************************************** */ + +#include + +// Use a runtime endian check just to be absolutely sure this code +// won't be responsible for any endian-related compile issues +// Call sha_init_endian to initialize before anything else! +static int sha_little_endian; + +static void sha_init_endian(void) +{ + static volatile int test = 1; + sha_little_endian = (*(char *)&test) ? 1 : 0; +} + +// Various types +typedef unsigned char Byte; +typedef unsigned int UInt32; +typedef unsigned long long int UInt64; + +typedef struct { + UInt32 state[8]; + UInt64 count; + Byte buffer[64]; +} CSha256; + +// Various macros +#define uintderef32(p) (*(UInt32 *)(p)) + +#ifdef _MSC_VER +#include +#pragma intrinsic(_rotr) +#define rotrFixed(x, n) _rotr((x), (n)) +#else +#define rotrFixed(x, n) (((x) >> (n)) | ((x) << (32 - (n)))) +#endif + +#ifndef __has_builtin +#define __has_builtin(x) 0 +#endif + +#if defined(_MSC_VER) && _MSC_VER >= 1300 +#pragma intrinsic(_byteswap_ulong) +#define byteswap32(v) _byteswap_ulong(v) +#elif (defined(__GNUC__) && (__GNUC__ > 4 || (__GNUC__ == 4 && __GNUC_MINOR__ >= 3))) || \ + (defined(__clang__) && __has_builtin(__builtin_bswap32)) +#define byteswap32(v) __builtin_bswap32(v) +#else +static unsigned int fsc_swap32(unsigned int value) +{ + value = ((value << 8) & 0xFF00FF00) | ((value >> 8) & 0xFF00FF); + return (value << 16) | (value >> 16); +} +#define byteswap32(v) fsc_swap32(v) +#endif + +#ifdef size_t +#undef size_t +#endif +#define size_t unsigned int + +// Place modifications to the original Sha256.c under this define +// to make them easier to identify +#define FSC_SHA + +/* ******************************************************************************** */ +// Sha256.c, lightly modified +/* ******************************************************************************** */ + +/* define it for speed optimization */ +#ifndef _SFX +#define _SHA256_UNROLL +#define _SHA256_UNROLL2 +#endif + +/* #define _SHA256_UNROLL2 */ + +void Sha256_Init(CSha256 *p) +{ +#ifdef FSC_SHA + sha_init_endian(); +#endif + p->state[0] = 0x6a09e667; + p->state[1] = 0xbb67ae85; + p->state[2] = 0x3c6ef372; + p->state[3] = 0xa54ff53a; + p->state[4] = 0x510e527f; + p->state[5] = 0x9b05688c; + p->state[6] = 0x1f83d9ab; + p->state[7] = 0x5be0cd19; + p->count = 0; +} + +#define S0(x) (rotrFixed(x, 2) ^ rotrFixed(x, 13) ^ rotrFixed(x, 22)) +#define S1(x) (rotrFixed(x, 6) ^ rotrFixed(x, 11) ^ rotrFixed(x, 25)) +#define s0(x) (rotrFixed(x, 7) ^ rotrFixed(x, 18) ^ (x >> 3)) +#define s1(x) (rotrFixed(x, 17) ^ rotrFixed(x, 19) ^ (x >> 10)) + +#define blk0(i) (W[i]) +#define blk2(i) (W[i] += s1(W[((i)-2) & 15]) + W[((i)-7) & 15] + s0(W[((i)-15) & 15])) + +#define Ch(x, y, z) (z ^ (x & (y ^ z))) +#define Maj(x, y, z) ((x & y) | (z & (x | y))) + +#ifdef _SHA256_UNROLL2 + +#define R(a, b, c, d, e, f, g, h, i) \ + h += S1(e) + Ch(e, f, g) + K[(i) + (j)] + (j ? blk2(i) : blk0(i)); \ + d += h; \ + h += S0(a) + Maj(a, b, c) + +#define RX_8(i) \ + R(a, b, c, d, e, f, g, h, i); \ + R(h, a, b, c, d, e, f, g, i + 1); \ + R(g, h, a, b, c, d, e, f, i + 2); \ + R(f, g, h, a, b, c, d, e, i + 3); \ + R(e, f, g, h, a, b, c, d, i + 4); \ + R(d, e, f, g, h, a, b, c, i + 5); \ + R(c, d, e, f, g, h, a, b, i + 6); \ + R(b, c, d, e, f, g, h, a, i + 7) + +#define RX_16 \ + RX_8(0); \ + RX_8(8); + +#else + +#define a(i) T[(0 - (i)) & 7] +#define b(i) T[(1 - (i)) & 7] +#define c(i) T[(2 - (i)) & 7] +#define d(i) T[(3 - (i)) & 7] +#define e(i) T[(4 - (i)) & 7] +#define f(i) T[(5 - (i)) & 7] +#define g(i) T[(6 - (i)) & 7] +#define h(i) T[(7 - (i)) & 7] + +#define R(i) \ + h(i) += S1(e(i)) + Ch(e(i), f(i), g(i)) + K[(i) + (j)] + (j ? blk2(i) : blk0(i)); \ + d(i) += h(i); \ + h(i) += S0(a(i)) + Maj(a(i), b(i), c(i)) + +#ifdef _SHA256_UNROLL + +#define RX_8(i) \ + R(i + 0); \ + R(i + 1); \ + R(i + 2); \ + R(i + 3); \ + R(i + 4); \ + R(i + 5); \ + R(i + 6); \ + R(i + 7); +#define RX_16 \ + RX_8(0); \ + RX_8(8); + +#else + +#define RX_16 \ + unsigned i; \ + for (i = 0; i < 16; i++) \ + { \ + R(i); \ + } + +#endif + +#endif + +static const UInt32 K[64] = {0x428a2f98, 0x71374491, 0xb5c0fbcf, 0xe9b5dba5, 0x3956c25b, 0x59f111f1, 0x923f82a4, + 0xab1c5ed5, 0xd807aa98, 0x12835b01, 0x243185be, 0x550c7dc3, 0x72be5d74, 0x80deb1fe, 0x9bdc06a7, 0xc19bf174, + 0xe49b69c1, 0xefbe4786, 0x0fc19dc6, 0x240ca1cc, 0x2de92c6f, 0x4a7484aa, 0x5cb0a9dc, 0x76f988da, 0x983e5152, + 0xa831c66d, 0xb00327c8, 0xbf597fc7, 0xc6e00bf3, 0xd5a79147, 0x06ca6351, 0x14292967, 0x27b70a85, 0x2e1b2138, + 0x4d2c6dfc, 0x53380d13, 0x650a7354, 0x766a0abb, 0x81c2c92e, 0x92722c85, 0xa2bfe8a1, 0xa81a664b, 0xc24b8b70, + 0xc76c51a3, 0xd192e819, 0xd6990624, 0xf40e3585, 0x106aa070, 0x19a4c116, 0x1e376c08, 0x2748774c, 0x34b0bcb5, + 0x391c0cb3, 0x4ed8aa4a, 0x5b9cca4f, 0x682e6ff3, 0x748f82ee, 0x78a5636f, 0x84c87814, 0x8cc70208, 0x90befffa, + 0xa4506ceb, 0xbef9a3f7, 0xc67178f2}; + +static void Sha256_WriteByteBlock(CSha256 *p) +{ + UInt32 W[16]; + unsigned j; + UInt32 *state; + +#ifdef _SHA256_UNROLL2 + UInt32 a, b, c, d, e, f, g, h; +#else + UInt32 T[8]; +#endif + + for (j = 0; j < 16; j += 4) + { + const Byte *ccc = p->buffer + j * 4; +#ifdef FSC_SHA + if (sha_little_endian) + { + W[j] = byteswap32(uintderef32(ccc)); + W[j + 1] = byteswap32(uintderef32(ccc + 4)); + W[j + 2] = byteswap32(uintderef32(ccc + 8)); + W[j + 3] = byteswap32(uintderef32(ccc + 12)); + } + else + { + W[j] = uintderef32(ccc); + W[j + 1] = uintderef32(ccc + 4); + W[j + 2] = uintderef32(ccc + 8); + W[j + 3] = uintderef32(ccc + 12); + } +#else + W[j] = GetBe32(ccc); + W[j + 1] = GetBe32(ccc + 4); + W[j + 2] = GetBe32(ccc + 8); + W[j + 3] = GetBe32(ccc + 12); +#endif + } + + state = p->state; + +#ifdef _SHA256_UNROLL2 + a = state[0]; + b = state[1]; + c = state[2]; + d = state[3]; + e = state[4]; + f = state[5]; + g = state[6]; + h = state[7]; +#else + for (j = 0; j < 8; j++) + T[j] = state[j]; +#endif + + for (j = 0; j < 64; j += 16) + { + RX_16 + } + +#ifdef _SHA256_UNROLL2 + state[0] += a; + state[1] += b; + state[2] += c; + state[3] += d; + state[4] += e; + state[5] += f; + state[6] += g; + state[7] += h; +#else + for (j = 0; j < 8; j++) + state[j] += T[j]; +#endif + + /* Wipe variables */ + /* memset(W, 0, sizeof(W)); */ + /* memset(T, 0, sizeof(T)); */ +} + +#undef S0 +#undef S1 +#undef s0 +#undef s1 + +void Sha256_Update(CSha256 *p, const Byte *data, size_t size) +{ + if (size == 0) + return; + + { + unsigned pos = (unsigned)p->count & 0x3F; + unsigned num; + + p->count += size; + + num = 64 - pos; + if (num > size) + { + memcpy(p->buffer + pos, data, size); + return; + } + + size -= num; + memcpy(p->buffer + pos, data, num); + data += num; + } + + for (;;) + { + Sha256_WriteByteBlock(p); + if (size < 64) + break; + size -= 64; + memcpy(p->buffer, data, 64); + data += 64; + } + + if (size != 0) + memcpy(p->buffer, data, size); +} + +void Sha256_Final(CSha256 *p, Byte *digest) +{ + unsigned pos = (unsigned)p->count & 0x3F; + unsigned i; + + p->buffer[pos++] = 0x80; + + while (pos != (64 - 8)) + { + pos &= 0x3F; + if (pos == 0) + Sha256_WriteByteBlock(p); + p->buffer[pos++] = 0; + } + + { + UInt64 numBits = (p->count << 3); +#ifdef FSC_SHA + if (sha_little_endian) + { + uintderef32(p->buffer + 64 - 8) = byteswap32((UInt32)(numBits >> 32)); + uintderef32(p->buffer + 64 - 4) = byteswap32((UInt32)numBits); + } + else + { + uintderef32(p->buffer + 64 - 8) = (UInt32)(numBits >> 32); + uintderef32(p->buffer + 64 - 4) = (UInt32)numBits; + } +#else + SetBe32(p->buffer + 64 - 8, (UInt32)(numBits >> 32)); + SetBe32(p->buffer + 64 - 4, (UInt32)(numBits)); +#endif + } + + Sha256_WriteByteBlock(p); + + for (i = 0; i < 8; i += 2) + { + UInt32 v0 = p->state[i]; + UInt32 v1 = p->state[i + 1]; +#ifdef FSC_SHA + if (sha_little_endian) + { + uintderef32(digest) = byteswap32(v0); + uintderef32(digest + 4) = byteswap32(v1); + } + else + { + uintderef32(digest) = v0; + uintderef32(digest + 4) = v1; + } +#else + SetBe32(digest, v0); + SetBe32(digest + 4, v1); +#endif + digest += 8; + } + + Sha256_Init(p); +} + +/* ******************************************************************************** */ +// Interface function +/* ******************************************************************************** */ + +void fsc_calculate_sha256(const char *data, unsigned int size, unsigned char *output) +{ + CSha256 csha; + Sha256_Init(&csha); + Sha256_Update(&csha, (const unsigned char *)data, size); + Sha256_Final(&csha, output); +} + +#endif // NEW_FILESYSTEM diff --git a/src/filesystem/fscore/fsc_shader.cpp b/src/filesystem/fscore/fsc_shader.cpp new file mode 100644 index 000000000..019a387ba --- /dev/null +++ b/src/filesystem/fscore/fsc_shader.cpp @@ -0,0 +1,143 @@ +/* +=========================================================================== +Copyright (C) 2017 Noah Metzger (chomenor@gmail.com) + +This file is part of Quake III Arena source code. + +Quake III Arena source code is free software; you can redistribute it +and/or modify it under the terms of the GNU General Public License as +published by the Free Software Foundation; either version 2 of the License, +or (at your option) any later version. + +Quake III Arena source code is distributed in the hope that it will be +useful, but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with Quake III Arena source code; if not, write to the Free Software +Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA +=========================================================================== +*/ + +#ifdef NEW_FILESYSTEM +#include "fscore.h" + +#define STACKPTR(pointer) (FSC_STACK_RETRIEVE(&fs->general_stack, pointer, 0)) // non-null + +/* ******************************************************************************** */ +// Shader Indexing +/* ******************************************************************************** */ + +static int index_shader_file_data( + fsc_filesystem_t *fs, fsc_stackptr_t source_file_ptr, char *data, fsc_errorhandler_t *eh) +{ + // Returns number of shaders indexed from file. + fsc_file_t *source_file = (fsc_file_t *)STACKPTR(source_file_ptr); + int shader_count = 0; + char *current_position = data; + int prefix_tokens; + char token[FSC_MAX_TOKEN_CHARS]; + + char shader_name[FSC_MAX_SHADER_NAME]; + unsigned int shader_start_position; + + unsigned int hash; + fsc_stackptr_t new_shader_ptr; + fsc_shader_t *new_shader; + + while (1) + { + prefix_tokens = 0; + + while (1) + { + // Load next token + shader_start_position = current_position - data; + fsc_COM_ParseExt(token, ¤t_position, 1); + if (!*token) + { + // We reached the end of the shader file. + if (prefix_tokens) + { + fsc_report_error(eh, FSC_ERROR_SHADERFILE, "shader file has extra tokens at end", source_file); + } + return shader_count; + } + + // Check for start of shader indicated by "{" + if (token[0] == '{' && !token[1]) + break; + + // Load potential shader name + fsc_strncpy_lower(shader_name, token, sizeof(shader_name)); + + ++prefix_tokens; + } + + if (!prefix_tokens) + { + fsc_report_error(eh, FSC_ERROR_SHADERFILE, "shader with no name", source_file); + continue; + } + + if (prefix_tokens > 1) + { + fsc_report_error(eh, FSC_ERROR_SHADERFILE, "shader with extra preceding tokens", source_file); + } + + // Skip to the end of the shader + if (fsc_SkipBracedSection(¤t_position, 1)) + { + fsc_report_error(eh, FSC_ERROR_SHADERFILE, "shader with no closing brace", source_file); + continue; + } + + // Allocate new shader + ++shader_count; + new_shader_ptr = fsc_stack_allocate(&fs->general_stack, sizeof(fsc_shader_t)); + new_shader = (fsc_shader_t *)STACKPTR(new_shader_ptr); + + // Copy data to new shader + new_shader->shader_name_ptr = + fsc_string_repository_getstring(shader_name, 1, &fs->string_repository, &fs->general_stack); + new_shader->source_file_ptr = source_file_ptr; + new_shader->start_position = shader_start_position; + new_shader->end_position = current_position - data; + + // Add shader to hash table + hash = fsc_string_hash(shader_name, 0); + fsc_hashtable_insert(new_shader_ptr, hash, &fs->shaders); + } + + return shader_count; +} + +int index_shader_file(fsc_filesystem_t *fs, fsc_stackptr_t source_file_ptr, fsc_errorhandler_t *eh) +{ + // Returns number of shaders indexed from file. + fsc_file_t *source_file = (fsc_file_t *)STACKPTR(source_file_ptr); + int shader_count; + + char *data = fsc_extract_file_allocated(fs, source_file, 0); + if (!data) + { + fsc_report_error(eh, FSC_ERROR_SHADERFILE, "failed to read shader file", source_file); + return 0; + } + + shader_count = index_shader_file_data(fs, source_file_ptr, data, eh); + fsc_free(data); + return shader_count; +} + +/* ******************************************************************************** */ +// Other +/* ******************************************************************************** */ + +int is_shader_enabled(fsc_filesystem_t *fs, const fsc_shader_t *shader) +{ + return fsc_is_file_enabled((const fsc_file_t *)STACKPTR(shader->source_file_ptr), fs); +} + +#endif // NEW_FILESYSTEM diff --git a/src/filesystem/fscore/fscore.h b/src/filesystem/fscore/fscore.h new file mode 100644 index 000000000..524041b75 --- /dev/null +++ b/src/filesystem/fscore/fscore.h @@ -0,0 +1,430 @@ +/* +=========================================================================== +Copyright (C) 2017 Noah Metzger (chomenor@gmail.com) + +This file is part of Quake III Arena source code. + +Quake III Arena source code is free software; you can redistribute it +and/or modify it under the terms of the GNU General Public License as +published by the Free Software Foundation; either version 2 of the License, +or (at your option) any later version. + +Quake III Arena source code is distributed in the hope that it will be +useful, but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with Quake III Arena source code; if not, write to the Free Software +Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA +=========================================================================== +*/ + +/* ******************************************************************************** */ +// Definitions +/* ******************************************************************************** */ + +#define FSC_CACHE_VERSION 11 + +#define FSC_MAX_QPATH 256 // Buffer size including null terminator +#define FSC_MAX_MODDIR 32 // Buffer size including null terminator + +#define FSC_MAX_TOKEN_CHARS 1024 // based on q_shared.h +#define FSC_MAX_SHADER_NAME FSC_MAX_TOKEN_CHARS + +/* ******************************************************************************** */ +// Misc (fsc_misc.c) +/* ******************************************************************************** */ + +typedef struct fsc_filesystem_s fsc_filesystem_t; + +// ***** Misc ***** + +unsigned int fsc_string_hash(const char *input1, const char *input2); +unsigned int fsc_fs_size_estimate(fsc_filesystem_t *fs); + +// ***** Standard Data Stream ***** + +typedef struct { + char *data; + unsigned int position; + unsigned int size; + int overflowed; +} fsc_stream_t; + +int fsc_read_stream_data(fsc_stream_t *stream, void *output, unsigned int length); +int fsc_write_stream_data(fsc_stream_t *stream, void *data, unsigned int length); +void fsc_stream_append_string_substituted(fsc_stream_t *stream, const char *string, const char *substitution_table); +void fsc_stream_append_string(fsc_stream_t *stream, const char *string); + +// ***** Standard Stack ***** + +typedef unsigned int fsc_stackptr_t; + +typedef struct { + unsigned int position; + // data added after structure +} fsc_stack_bucket_t; + +typedef struct { + fsc_stack_bucket_t **buckets; + int buckets_position; + int buckets_size; +} fsc_stack_t; + +void fsc_stack_initialize(fsc_stack_t *stack); +fsc_stackptr_t fsc_stack_allocate(fsc_stack_t *stack, unsigned int size); +void *fsc_stack_retrieve(const fsc_stack_t *stack, const fsc_stackptr_t pointer, int allow_null, + const char *caller, const char *expression); +void fsc_stack_free(fsc_stack_t *stack); +unsigned int fsc_stack_get_export_size(fsc_stack_t *stack); +int fsc_stack_export(fsc_stack_t *stack, fsc_stream_t *stream); +int fsc_stack_import(fsc_stack_t *stack, fsc_stream_t *stream); + +#define FSC_STACK_RETRIEVE(stack, pointer, allow_null) \ + fsc_stack_retrieve(stack, pointer, allow_null, __func__, #pointer) + +// ***** Standard Hash Table ***** + +#define FSC_HASHTABLE_MAX_BUCKETS (10 << 20) + +// This needs to be the first field of each entry of the hash table, so each entry can be casted to this type +typedef struct { + fsc_stackptr_t next; +} fsc_hashtable_entry_t; + +typedef struct { + fsc_stackptr_t *buckets; + fsc_stack_t *stack; + int bucket_count; + int utilization; +} fsc_hashtable_t; + +typedef struct { + fsc_stack_t *stack; + fsc_stackptr_t *next_ptr; +} fsc_hashtable_iterator_t; + +void fsc_hashtable_initialize(fsc_hashtable_t *ht, fsc_stack_t *stack, int bucket_count); +void fsc_hashtable_open(fsc_hashtable_t *ht, unsigned int hash, fsc_hashtable_iterator_t *iterator); +fsc_stackptr_t fsc_hashtable_next(fsc_hashtable_iterator_t *iterator); +void fsc_hashtable_insert(fsc_stackptr_t entry_ptr, unsigned int hash, fsc_hashtable_t *ht); +void fsc_hashtable_free(fsc_hashtable_t *ht); +unsigned int fsc_hashtable_get_export_size(fsc_hashtable_t *ht); +int fsc_hashtable_export(fsc_hashtable_t *ht, fsc_stream_t *stream); +int fsc_hashtable_import(fsc_hashtable_t *ht, fsc_stack_t *stack, fsc_stream_t *stream); + +// ***** Standard String Repository ***** + +typedef struct stringrepository_entry_s { + // Hash table compliance + fsc_hashtable_entry_t hte; + // data added after structure +} stringrepository_entry_t; + +fsc_stackptr_t fsc_string_repository_getentry(const char *input, int allocate, fsc_hashtable_t *string_repository, fsc_stack_t *stack); // stringrepository_entry_t +fsc_stackptr_t fsc_string_repository_getstring(const char *input, int allocate, fsc_hashtable_t *string_repository, fsc_stack_t *stack); // char * + +// ***** Qpath Handling ***** + +const char *fsc_get_qpath_conversion_table(void); +int fsc_process_qpath(const char *input, char *buffer, const char **qp_dir, const char **qp_name, const char **qp_ext); +unsigned int fsc_get_leading_directory(const char *input, char *buffer, unsigned int buffer_length, const char **remainder); + +// ***** Error Handling ***** + +#define FSC_ERROR_GENERAL 0 +#define FSC_ERROR_EXTRACT 1 +#define FSC_ERROR_PK3FILE 2 // current_element: fsc_file_t (fsc_file_direct_t) +#define FSC_ERROR_SHADERFILE 3 // current_element: fsc_file_t +#define FSC_ERROR_CROSSHAIRFILE 4 // current_element: fsc_file_t + +#define FSC_ASSERT(expression) if(!(expression)) fsc_fatal_error_tagged("assertion failed", __func__, #expression); + +typedef struct { + void (*handler)(int id, const char *msg, void *current_element, void *context); + void *context; +} fsc_errorhandler_t; + +void fsc_report_error(fsc_errorhandler_t *errorhandler, int id, const char *msg, void *current_element); +void fsc_register_fatal_error_handler(void (*handler)(const char *msg)); +void fsc_fatal_error(const char *msg); +void fsc_fatal_error_tagged(const char *msg, const char *caller, const char *expression); + +/* ******************************************************************************** */ +// Game Parsing Support (fsc_gameparse.c) +/* ******************************************************************************** */ + +void fsc_SkipRestOfLine ( char **data ); +char *fsc_COM_ParseExt( char *com_token, char **data_p, int allowLineBreaks ); +int fsc_SkipBracedSection(char **program, int depth); + +/* ******************************************************************************** */ +// Hash Calculation (fsc_md4.c / fsc_sha256.c) +/* ******************************************************************************** */ + +unsigned int fsc_block_checksum(const void *buffer, int length); +void fsc_calculate_sha256(const char *data, unsigned int size, unsigned char *output); + +/* ******************************************************************************** */ +// OS Library Interface (fsc_os.c) +/* ******************************************************************************** */ + +typedef struct { + void *os_path; + char *qpath_with_mod_dir; + unsigned int os_timestamp; + unsigned int filesize; +} iterate_data_t; + +typedef enum { + FSC_SEEK_SET, + FSC_SEEK_CUR, + FSC_SEEK_END +} fsc_seek_type_t; + +void *fsc_string_to_os_path(const char *path); // WARNING: Result must be freed by caller using fsc_free!!! +char *fsc_os_path_to_string(const void *os_path); // WARNING: Result must be freed by caller using fsc_free!!! +int fsc_os_path_size(const void *os_path); +int fsc_compare_os_path(const void *path1, const void *path2); + +void iterate_directory(void *search_os_path, void (operation)(iterate_data_t *file_data, + void *iterate_context), void *iterate_context); + +void fsc_error_abort(const char *msg); +int fsc_rename_file(void *source_os_path, void *target_os_path); +int fsc_delete_file(void *os_path); +int fsc_mkdir(void *os_path); +void *fsc_open_file(const void *os_path, const char *mode); +void fsc_fclose(void *fp); +unsigned int fsc_fread(void *dest, int size, void *fp); +unsigned int fsc_fwrite(const void *src, int size, void *fp); +void fsc_fflush(void *fp); +int fsc_fseek(void *fp, int offset, fsc_seek_type_t type); +int fsc_fseek_set(void *fp, unsigned int offset); +unsigned int fsc_ftell(void *fp); +void fsc_memcpy(void *dst, const void *src, unsigned int size); +int fsc_memcmp(const void *str1, const void *str2, unsigned int size); +void fsc_memset(void *dst, int value, unsigned int size); +void fsc_strncpy(char *dst, const char *src, unsigned int size); +void fsc_strncpy_lower(char *dst, const char *src, unsigned int size); +int fsc_strcmp(const char *str1, const char *str2); +int fsc_stricmp(const char *str1, const char *str2); +int fsc_strlen(const char *str); +void *fsc_malloc(unsigned int size); +void *fsc_calloc(unsigned int size); +void fsc_free(void *allocation); + +/* ******************************************************************************** */ +// Main Filesystem (fsc_main.c) +/* ******************************************************************************** */ + +// ***** Core Filesystem Structures ***** + +#define FSC_SOURCETYPE_DIRECT 1 +#define FSC_SOURCETYPE_PK3 2 + +#define FSC_FILEFLAG_LINKED_CONTENT 1 // This file has other content like pk3 contents or shaders linked to it +#define FSC_FILEFLAG_DLPK3 2 // This pk3 is located in a download directory + +typedef struct fsc_file_s { + // Hash table compliance + fsc_hashtable_entry_t hte; + + // Identification + // Note: The character encoding for qpaths is currently not standardized for values outside the ASCII range (val > 127) + // It depends on the encoding used by the OS library / pk3 file, which may be UTF-8, CP-1252, or something else + // Currently most content just uses ASCII characters + fsc_stackptr_t qp_dir_ptr; // null for no directory + fsc_stackptr_t qp_name_ptr; // should not be null + fsc_stackptr_t qp_ext_ptr; // null for no extension + + unsigned int filesize; + fsc_stackptr_t contents_cache; // pointer to file data if cached, null otherwise + fsc_stackptr_t next_in_directory; // Iteration + unsigned short flags; + unsigned short sourcetype; +} fsc_file_t; + +typedef struct { + fsc_file_t f; + + // Enable / Disable + int refresh_count; + + int source_dir_id; + fsc_stackptr_t os_path_ptr; + unsigned int os_timestamp; + fsc_stackptr_t qp_mod_ptr; + + fsc_stackptr_t pk3dir_ptr; // null if file is not part of a pk3dir + unsigned int pk3_hash; // null if file is not a valid pk3 + + // For resource tallies + unsigned int pk3_subfile_count; + unsigned int shader_file_count; + unsigned int shader_count; +} fsc_file_direct_t; + +typedef struct { + fsc_file_t f; + fsc_stackptr_t source_pk3; // fsc_file_direct_t + unsigned int header_position; + unsigned int compressed_size; + short compression_method; +} fsc_file_frompk3_t; + +typedef struct { + fsc_hashtable_entry_t hte; + fsc_stackptr_t pk3; +} fsc_pk3_hash_map_entry_t; + +typedef struct { + // Identifies the sourcetype - 1 and 2 are reserved for FSC_SOURCETYPE_DIRECT and FSC_SOURCETYPE_PK3 + int sourcetype_id; + + // Returns 1 if active, 0 otherwise. + int (*is_file_active)(const fsc_file_t *file, const fsc_filesystem_t *fs); + + // Should always return a valid static string. + const char *(*get_mod_dir)(const fsc_file_t *file, const fsc_filesystem_t *fs); + + // Buffer should be length file->filesize + // Returns 0 on success, 1 on failure. + int (*extract_data)(const fsc_file_t *file, char *buffer, const fsc_filesystem_t *fs, fsc_errorhandler_t *eh); +} fsc_sourcetype_t; + +typedef struct { + // Useful for info print messages and to determine + // a reasonable size for building exported hashtables + int valid_pk3_count; + int pk3_subfile_count; + int shader_file_count; + int shader_count; + int total_file_count; + int cacheable_file_count; +} fsc_stats_t; + +#define FSC_CUSTOM_SOURCETYPE_COUNT 2 +typedef struct fsc_filesystem_s { + // Support + fsc_stack_t general_stack; + fsc_hashtable_t string_repository; + + // Main Filesystem + fsc_hashtable_t files; + int refresh_count; + + // Iteration + fsc_hashtable_t directories; + + // Shaders + fsc_hashtable_t shaders; + + // Crosshairs + fsc_hashtable_t crosshairs; + + // PK3 Hash Lookup - Useful to determine files needed to download + fsc_hashtable_t pk3_hash_lookup; + + // Custom Sourcetypes - Can be used for special applications + fsc_sourcetype_t custom_sourcetypes[FSC_CUSTOM_SOURCETYPE_COUNT]; + + // Stats + fsc_stats_t total_stats; + fsc_stats_t active_stats; + fsc_stats_t new_stats; +} fsc_filesystem_t; + +// ***** Functions ***** + +const fsc_file_direct_t *fsc_get_base_file(const fsc_file_t *file, const fsc_filesystem_t *fs); +int fsc_extract_file(const fsc_file_t *file, char *buffer, const fsc_filesystem_t *fs, fsc_errorhandler_t *eh); +char *fsc_extract_file_allocated(fsc_filesystem_t *index, fsc_file_t *file, fsc_errorhandler_t *eh); + +int fsc_is_file_enabled(const fsc_file_t *file, const fsc_filesystem_t *fs); +const char *fsc_get_mod_dir(const fsc_file_t *file, const fsc_filesystem_t *fs); +void fsc_file_to_stream(const fsc_file_t *file, fsc_stream_t *stream, const fsc_filesystem_t *fs, + int include_mod, int include_pk3_origin); + +void fsc_register_file(fsc_stackptr_t file_ptr, fsc_filesystem_t *fs, fsc_errorhandler_t *eh); +void fsc_load_file(int source_dir_id, const void *os_path, const char *mod_dir, const char *pk3dir_name, + const char *qp_dir, const char *qp_name, const char *qp_ext, unsigned int os_timestamp, unsigned int filesize, + fsc_filesystem_t *fs, fsc_errorhandler_t *eh); +void fsc_load_file_full_path(int source_dir_id, const void *os_path, const char *full_qpath, unsigned int os_timestamp, + unsigned int filesize, fsc_filesystem_t *fs, fsc_errorhandler_t *eh); + +void fsc_filesystem_initialize(fsc_filesystem_t *fs); +void fsc_filesystem_free(fsc_filesystem_t *fs); +void fsc_filesystem_reset(fsc_filesystem_t *fs); +void fsc_load_directory(fsc_filesystem_t *fs, void *os_path, int source_dir_id, fsc_errorhandler_t *eh); + +/* ******************************************************************************** */ +// PK3 Handling (fsc_pk3.c) +/* ******************************************************************************** */ + +// receive_hash_data is used for standalone hash calculation operations, +// and should be nulled during normal filesystem loading +void fsc_load_pk3(void *os_path, fsc_filesystem_t *fs, fsc_stackptr_t sourcefile_ptr, fsc_errorhandler_t *eh, + void (*receive_hash_data)(void *context, char *data, int size), void *receive_hash_data_context ); +void register_pk3_hash_lookup_entry(fsc_stackptr_t pk3_file_ptr, fsc_hashtable_t *pk3_hash_lookup, fsc_stack_t *stack); +void *fsc_pk3_handle_open(const fsc_file_frompk3_t *file, int input_buffer_size, const fsc_filesystem_t *fs, fsc_errorhandler_t *eh); +void fsc_pk3_handle_close(void *handle); +unsigned int fsc_pk3_handle_read(void *handle, char *buffer, unsigned int length); +extern fsc_sourcetype_t pk3_sourcetype; + +/* ******************************************************************************** */ +// Shader Lookup (fsc_shader.c) +/* ******************************************************************************** */ + +typedef struct fsc_shader_s { + // Hash table compliance + fsc_hashtable_entry_t hte; + + fsc_stackptr_t shader_name_ptr; + + fsc_stackptr_t source_file_ptr; + unsigned int start_position; + unsigned int end_position; +} fsc_shader_t; + +int index_shader_file(fsc_filesystem_t *fs, fsc_stackptr_t source_file_ptr, fsc_errorhandler_t *eh); +int is_shader_enabled(fsc_filesystem_t *fs, const fsc_shader_t *shader); + +/* ******************************************************************************** */ +// Crosshair Lookup (fsc_crosshair.c) +/* ******************************************************************************** */ + +typedef struct { + // Hash table compliance + fsc_hashtable_entry_t hte; + + unsigned int hash; + fsc_stackptr_t source_file_ptr; +} fsc_crosshair_t; + +int index_crosshair(fsc_filesystem_t *fs, fsc_stackptr_t source_file_ptr, fsc_errorhandler_t *eh); +int is_crosshair_enabled(fsc_filesystem_t *fs, const fsc_crosshair_t *crosshair); + +/* ******************************************************************************** */ +// Iteration (fsc_iteration.c) +/* ******************************************************************************** */ + +typedef struct { + // Hash table compliance + fsc_hashtable_entry_t hte; + + fsc_stackptr_t qp_dir_ptr; // string repository + fsc_stackptr_t peer_directory; + fsc_stackptr_t sub_file; + fsc_stackptr_t sub_directory; +} fsc_directory_t; + +void fsc_iteration_register_file(fsc_stackptr_t file_ptr, fsc_hashtable_t *directories, fsc_hashtable_t *string_repository, fsc_stack_t *stack); + +/* ******************************************************************************** */ +// Index Cache (fsc_index.c) +/* ******************************************************************************** */ + +int fsc_cache_export_file(fsc_filesystem_t *source_fs, void *os_path, fsc_errorhandler_t *eh); +int fsc_cache_import_file(void *os_path, fsc_filesystem_t *target_fs, fsc_errorhandler_t *eh); diff --git a/src/filesystem/fslocal.h b/src/filesystem/fslocal.h new file mode 100644 index 000000000..59eaeeeb2 --- /dev/null +++ b/src/filesystem/fslocal.h @@ -0,0 +1,36 @@ +/* +=========================================================================== +Copyright (C) 1999-2005 Id Software, Inc. +Copyright (C) 2017 Noah Metzger (chomenor@gmail.com) + +This file is part of Quake III Arena source code. + +Quake III Arena source code is free software; you can redistribute it +and/or modify it under the terms of the GNU General Public License as +published by the Free Software Foundation; either version 2 of the License, +or (at your option) any later version. + +Quake III Arena source code is distributed in the hope that it will be +useful, but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with Quake III Arena source code; if not, write to the Free Software +Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA +=========================================================================== +*/ + +// Enable the internal filesystem defs in fspublic.h when it gets included via files.h +#define FSLOCAL +#define STANDALONE + +#include "fscore/fscore.h" +#include "qcommon/cmd.h" +#include "qcommon/cvar.h" +#include "qcommon/files.h" +#include "sys/sys_shared.h" +#include "../qcommon/q_shared.h" +#include "../qcommon/qcommon.h" + +#define qboolean bool diff --git a/src/filesystem/fspublic.h b/src/filesystem/fspublic.h new file mode 100644 index 000000000..bfd47f431 --- /dev/null +++ b/src/filesystem/fspublic.h @@ -0,0 +1,495 @@ +/* +=========================================================================== +Copyright (C) 1999-2005 Id Software, Inc. +Copyright (C) 2017 Noah Metzger (chomenor@gmail.com) + +This file is part of Quake III Arena source code. + +Quake III Arena source code is free software; you can redistribute it +and/or modify it under the terms of the GNU General Public License as +published by the Free Software Foundation; either version 2 of the License, +or (at your option) any later version. + +Quake III Arena source code is distributed in the hope that it will be +useful, but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with Quake III Arena source code; if not, write to the Free Software +Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA +=========================================================================== +*/ + +#include +#include "../qcommon/qcommon.h" +#include "../qcommon/q_shared.h" + +/* ******************************************************************************** */ +// Common Definitions +/* ******************************************************************************** */ + +#ifdef FSLOCAL +// Versioned pk3 search locations +#define BASEGAME_1_3 "trem13" +#define BASEGAME_GPP "gpp" +#define BASEGAME_1_1 "base" + +// User mod directory; overrides the regular search locations +#define BASEGAME_OVERRIDE "basemod" + +// Store location for saved demos, screenshots, configs, condumps, etc. +// Can be modified on command line via fs_basegame cvar +#define FS_BASEGAME_DEFAULT "trem13" + +// Name of core game dll (full location is {BASEGAME_1_3}/{FS_CORE_GAME_DLL_NAME}.{DLL_EXT}) +#define FS_CORE_GAME_DLL_NAME "core_game" +#endif + +#define qboolean bool +typedef FS_Origin fsOrigin_t; +typedef FS_Mode fsMode_t; + +#define DEF_PUBLIC(f) f; +#ifdef FSLOCAL +#define DEF_LOCAL(f) f; +#else +#define DEF_LOCAL(f) +#endif + +#ifdef DEDICATED +#define Q3CONFIG_CFG "autogen_server.cfg" +#else +#define Q3CONFIG_CFG "autogen.cfg" +#endif + +#ifdef WIN32 +#define SYSTEM_NEWLINE "\r\n" +#else +#define SYSTEM_NEWLINE "\n" +#endif + +typedef struct fsc_file_s fsc_file_t; +typedef struct fsc_shader_s fsc_shader_t; + +typedef enum { + FS_CONFIGTYPE_NONE, + FS_CONFIGTYPE_DEFAULT, + FS_CONFIGTYPE_SETTINGS +} fs_config_type_t; + +#ifdef FSLOCAL +enum FS_Profile { + FS_PROFILE_UNSET, + FS_PROFILE_1_1, + FS_PROFILE_GPP, + FS_PROFILE_1_3 +}; + +typedef struct { + const char *name; + const char *path; + qboolean active; + qboolean auxiliary; +} fs_source_directory_t; + +typedef struct fs_hashtable_entry_s { + struct fs_hashtable_entry_s *next; +} fs_hashtable_entry_t; + +typedef struct { + fs_hashtable_entry_t **buckets; + int bucket_count; + int element_count; +} fs_hashtable_t; + +typedef struct { + fs_hashtable_t *ht; + int current_bucket; + int bucket_limit; + fs_hashtable_entry_t *current_entry; +} fs_hashtable_iterator_t; + +typedef struct pk3_list_entry_s { + fs_hashtable_entry_t hte; + unsigned int hash; + int position; +} pk3_list_entry_t; + +typedef struct { + fs_hashtable_t ht; +} pk3_list_t; + +#define STACKPTR(pointer) ( FSC_STACK_RETRIEVE(&fs.general_stack, pointer, 0) ) // non-null +#define STACKPTRN(pointer) ( FSC_STACK_RETRIEVE(&fs.general_stack, pointer, 1) ) // null allowed +#endif + +/* ******************************************************************************** */ +// Main +/* ******************************************************************************** */ + +DEF_LOCAL( extern cvar_t *fs_dirs ) +DEF_LOCAL( extern cvar_t *fs_mod_settings ) +DEF_LOCAL( extern cvar_t *fs_index_cache ) +DEF_LOCAL( extern cvar_t *fs_read_inactive_mods ) +DEF_LOCAL( extern cvar_t *fs_list_inactive_mods ) +DEF_LOCAL( extern cvar_t *fs_download_manifest ) +DEF_LOCAL( extern cvar_t *fs_pure_manifest ) +DEF_LOCAL( extern cvar_t *fs_redownload_across_mods ) +DEF_LOCAL( extern cvar_t *fs_full_pure_validation ) +DEF_LOCAL( extern cvar_t *fs_saveto_dlfolder ) +DEF_LOCAL( extern cvar_t *fs_restrict_dlfolder ) +DEF_LOCAL( extern cvar_t *fs_auto_refresh ) + +DEF_LOCAL( extern cvar_t *fs_debug_state ) +DEF_LOCAL( extern cvar_t *fs_debug_refresh ) +DEF_LOCAL( extern cvar_t *fs_debug_fileio ) +DEF_LOCAL( extern cvar_t *fs_debug_lookup ) +DEF_LOCAL( extern cvar_t *fs_debug_references ) +DEF_LOCAL( extern cvar_t *fs_debug_filelist ) + +DEF_LOCAL( extern cvar_t *fs_basegame ) +DEF_LOCAL( extern cvar_t *fs_force_profile ) + +#ifdef FSLOCAL +#define FS_MAX_SOURCEDIRS 16 +#endif + +DEF_LOCAL( extern fs_source_directory_t fs_sourcedirs[FS_MAX_SOURCEDIRS] ) +DEF_LOCAL( extern qboolean fs_read_only ) + +DEF_LOCAL( extern fsc_filesystem_t fs ) + +DEF_LOCAL( extern cvar_t *fs_game ) +DEF_LOCAL( extern char current_mod_dir[FSC_MAX_MODDIR] ) +DEF_LOCAL( extern const fsc_file_direct_t *current_map_pk3 ) +DEF_LOCAL( extern int checksum_feed ) + +DEF_LOCAL( extern int connected_server_sv_pure ) +DEF_LOCAL( extern pk3_list_t connected_server_pure_list ) +DEF_LOCAL( extern FS_Profile fs_connected_server_profile ) + +// State Accessors +DEF_PUBLIC( const char *FS_GetCurrentGameDir(void) ) +DEF_PUBLIC( const char *fs_pid_file_directory(void) ) +DEF_PUBLIC( qboolean FS_Initialized( void ) ) +DEF_LOCAL( int fs_connected_server_pure_state(void) ) +DEF_LOCAL( FS_Profile fs_current_profile(void) ) + +// State Modifiers +DEF_PUBLIC( void fs_register_current_map(const char *name) ) +DEF_PUBLIC( void fs_set_connected_server_profile(int alternateProtocol) ) +DEF_PUBLIC( void fs_set_connected_server_sv_pure_value(int sv_pure) ) +DEF_PUBLIC( void FS_PureServerSetLoadedPaks(const char *hash_list, const char *name_list) ) +DEF_PUBLIC( void fs_disconnect_cleanup(void) ) +DEF_PUBLIC( void fs_set_mod_dir(const char *value, qboolean move_pid) ) +DEF_PUBLIC( qboolean FS_ConditionalRestart(int checksumFeed, qboolean disconnect) ) + +// Filesystem Refresh +DEF_LOCAL( void fs_refresh(qboolean quiet) ) +DEF_LOCAL( void fs_refresh_auto_ext(qboolean ignore_cvar, qboolean quiet) ) +DEF_PUBLIC( void fs_refresh_auto(void) ) + +// Filesystem Initialization +DEF_LOCAL( void fs_indexcache_write(void) ) +DEF_PUBLIC( void fs_startup(void) ) + +/* ******************************************************************************** */ +// Lookup +/* ******************************************************************************** */ + +#define LOOKUPFLAG_ENABLE_DDS 1 // Enable dds format for image lookups. Must match value in tr_public.h! +#define LOOKUPFLAG_IGNORE_PURE_LIST 2 // Ignore pure list entirely (allow all files AND ignore ordering) +#define LOOKUPFLAG_PURE_ALLOW_DIRECT_SOURCE 4 // Allow files on disk when pure +#define LOOKUPFLAG_IGNORE_CURRENT_MAP 8 // Ignore current map sort criteria +#define LOOKUPFLAG_DIRECT_SOURCE_ONLY 16 // Only allow files on disk +#define LOOKUPFLAG_PK3_SOURCE_ONLY 32 // Only allow files in pk3s +#define LOOKUPFLAG_SETTINGS_FILE 64 // Apply fs_mod_settings for auto-executed settings files (e.g. autogen.cfg, autoexec.cfg) +#define LOOKUPFLAG_NO_DOWNLOAD_FOLDER 128 // Don't allow files from download folder +#define LOOKUPFLAG_PRIORITIZE_FS_BASEGAME 256 // (Tremulous) Prioritize fs_basegame over the other hardcoded base directories + +DEF_LOCAL( void debug_resource_comparison(int resource1_position, int resource2_position) ) +DEF_PUBLIC( const fsc_file_t *fs_general_lookup(const char *name, int lookup_flags, qboolean debug) ) +DEF_PUBLIC( const fsc_shader_t *fs_shader_lookup(const char *name, int lookup_flags, qboolean debug) ) +DEF_PUBLIC( const fsc_file_t *fs_image_lookup(const char *name, int lookup_flags, qboolean debug) ) +DEF_PUBLIC( const fsc_file_t *fs_sound_lookup(const char *name, int lookup_flags, qboolean debug) ) +DEF_PUBLIC( const fsc_file_t *fs_vm_lookup(const char *name, qboolean qvm_only, qboolean debug, qboolean *is_dll_out) ) + +/* ******************************************************************************** */ +// File Listing +/* ******************************************************************************** */ + +#define FLISTFLAG_IGNORE_TAPAK0 1 // Ignore missionpak pak0.pk3 (to keep incompatible models out of model list) +#define FLISTFLAG_IGNORE_PURE_LIST 2 // Ignore pure list entirely (allow all files AND ignore ordering) +#define FLISTFLAG_PURE_ALLOW_DIRECT_SOURCE 4 // Allow files on disk when pure + +DEF_PUBLIC( void FS_FreeFileList( char **list ) ) +DEF_LOCAL( char **FS_FlagListFilteredFiles(const char *path, const char *extension, const char *filter, + int *numfiles_out, int flags) ) +DEF_PUBLIC( char **FS_ListFiles( const char *path, const char *extension, int *numfiles ) ) +DEF_PUBLIC( int FS_GetFileList( const char *path, const char *extension, char *listbuf, int bufsize ) ) +DEF_PUBLIC( int FS_GetFilteredFiles( const char *path, const char *extension, const char *filter, char *listbuf, int bufsize ) ) + +/* ******************************************************************************** */ +// File IO +/* ******************************************************************************** */ + +// Path Generation Flags +#define FS_MAX_PATH 512 +#define FS_NO_SANITIZE 1 +#define FS_CREATE_DIRECTORIES 2 +#define FS_CREATE_DIRECTORIES_FOR_FILE 4 +#define FS_ALLOW_DIRECTORIES 8 +#define FS_ALLOW_PK3 16 +#define FS_ALLOW_DLL 32 +#define FS_ALLOW_SPECIAL_CFG 64 + +// Path Generation Functions +DEF_LOCAL( unsigned int fs_generate_path_sourcedir(int source_dir_id, const char *path1, const char *path2, + int path1_flags, int path2_flags, char *target, unsigned int target_size) ) +DEF_PUBLIC( unsigned int fs_generate_path(const char *path1, const char *path2, const char *path3, + int path1_flags, int path2_flags, int path3_flags, + char *target, unsigned int target_size) ) +DEF_PUBLIC( unsigned int fs_generate_path_writedir(const char *path1, const char *path2, + int path1_flags, int path2_flags, char *target, unsigned int target_size) ) + +// Direct file access +DEF_LOCAL( void *fs_open_file(const char *path, const char *mode) ) +DEF_LOCAL( void fs_rename_file(const char *source, const char *target) ) +DEF_LOCAL( void fs_delete_file(const char *path) ) +DEF_PUBLIC( void FS_HomeRemove( const char *homePath ) ) +DEF_LOCAL( qboolean FS_FileInPathExists(const char *testpath) ) +DEF_PUBLIC( qboolean FS_FileExists(const char *file) ) +DEF_PUBLIC( qboolean FS_FileExistsInBaseGame(const char *file) ) + +// File read cache +DEF_PUBLIC( void fs_cache_initialize(void) ) +DEF_PUBLIC( void fs_advance_cache_stage(void) ) +DEF_LOCAL( void fs_readcache_debug(void) ) + +// Data reading +DEF_PUBLIC( char *fs_read_data(const fsc_file_t *file, const char *path, unsigned int *size_out, const char *calling_function) ) +DEF_PUBLIC( void fs_free_data(char *data) ) +DEF_PUBLIC( char *fs_read_shader(const fsc_shader_t *shader) ) + +// Direct read handle operations +DEF_PUBLIC( fileHandle_t fs_direct_read_handle_open(const fsc_file_t *file, const char *path, unsigned int *size_out) ) + +// Pipe files +DEF_PUBLIC( fileHandle_t FS_FCreateOpenPipeFile( const char *filename ) ) + +// Handle owners +typedef enum { + FS_HANDLEOWNER_SYSTEM, + FS_HANDLEOWNER_CGAME, + FS_HANDLEOWNER_UI, + FS_HANDLEOWNER_QAGAME +} fs_handle_owner_t; + +// Common handle operations +DEF_PUBLIC( void fs_handle_close(fileHandle_t handle) ) +DEF_PUBLIC( void fs_close_all_handles(void) ) +DEF_PUBLIC( fs_handle_owner_t fs_handle_get_owner(fileHandle_t handle) ) +DEF_LOCAL( void fs_print_handle_list(void) ) +DEF_PUBLIC( void fs_close_owner_handles(fs_handle_owner_t owner) ) + +// Journal Data File Functions +DEF_LOCAL( void fs_write_journal_data(const char *data, unsigned int length) ) +DEF_LOCAL( char *fs_read_journal_data(void) ) + +// Config file functions +DEF_PUBLIC( fileHandle_t fs_open_settings_file_write(const char *filename) ) + +// Data reading operations +DEF_PUBLIC( long FS_ReadFile(const char *qpath, void **buffer) ) +DEF_PUBLIC( void FS_FreeFile(void *buffer) ) + +// "Read-back" tracking +DEF_LOCAL( void fs_readback_tracker_reset(void) ) + +// FS_FOpenFile functions +DEF_PUBLIC( long FS_FOpenFileRead(const char *filename, fileHandle_t *file, qboolean uniqueFILE) ) +DEF_PUBLIC( fileHandle_t FS_FOpenFileWrite(const char *filename) ) +DEF_PUBLIC( fileHandle_t FS_FOpenFileAppend(const char *filename) ) +DEF_PUBLIC( int FS_FOpenFileByModeOwner(const char *qpath, fileHandle_t *f, fsMode_t mode, fs_handle_owner_t owner) ) +DEF_PUBLIC( int FS_FOpenFileByMode(const char *qpath, fileHandle_t *f, fsMode_t mode) ) + +// Misc handle operations +DEF_PUBLIC( long FS_SV_FOpenFileRead(const char *filename, fileHandle_t *fp) ) +DEF_PUBLIC( fileHandle_t FS_SV_FOpenFileWrite(const char *filename) ) +DEF_PUBLIC( void FS_FCloseFile(fileHandle_t f) ) +DEF_PUBLIC( int FS_Read(void *buffer, int len, fileHandle_t f) ) +DEF_PUBLIC( int FS_Read2(void *buffer, int len, fileHandle_t f) ) +DEF_PUBLIC( int FS_Write(const void *buffer, int len, fileHandle_t h) ) +DEF_PUBLIC( int FS_Seek(fileHandle_t f, long offset, int origin) ) +DEF_PUBLIC( int FS_FTell(fileHandle_t f) ) +DEF_PUBLIC( void FS_Flush(fileHandle_t f) ) +DEF_PUBLIC( void FS_ForceFlush(fileHandle_t f) ) +DEF_PUBLIC( void FS_SV_Rename(const char *from, const char *to, qboolean safe) ) +DEF_PUBLIC( void FS_WriteFile( const char *qpath, const void *buffer, int size ) ) + +/* ******************************************************************************** */ +// Commands +/* ******************************************************************************** */ + +DEF_LOCAL( void fs_register_commands(void) ) + +/* ******************************************************************************** */ +// Download +/* ******************************************************************************** */ + +// Download List Handling +DEF_PUBLIC( void fs_advance_download(void) ) +DEF_PUBLIC( void fs_print_download_list(void) ) +DEF_PUBLIC( void fs_register_download_list(const char *hash_list, const char *name_list) ) + +// Attempted Download Tracking +DEF_PUBLIC( void fs_register_current_download_attempt(qboolean http) ) +DEF_PUBLIC( void fs_clear_attempted_downloads(void) ) + +// Download List Advancement +DEF_PUBLIC( void fs_advance_next_needed_download(qboolean curl_disconnected) ) +DEF_PUBLIC( qboolean fs_get_current_download_info(char **local_name_out, char **remote_name_out, + qboolean *curl_already_attempted_out) ) +DEF_PUBLIC( int fs_estimate_remaining_downloads(void) ) + +// Download Completion +DEF_PUBLIC( void fs_finalize_download(void) ) + +/* ******************************************************************************** */ +// Reference List Handling +/* ******************************************************************************** */ + +struct FS_ReferenceProfile { + // Defines reference-related constants for a particular client profile + std::string profile_description; + std::string download_hashes_cvar; + std::string download_names_cvar; + std::string pure_hashes_cvar; + std::string pure_names_cvar; + FS_ReferenceProfile(clientProfile_t profile); +}; + +class FS_DownloadMapShared; + +DEF_LOCAL( void fs_register_reference(const fsc_file_t *file) ) +DEF_PUBLIC( void FS_ClearPakReferences( int flags ) ) +DEF_PUBLIC( const char *FS_ReferencedPakNames( void ) ) +DEF_PUBLIC( const char *FS_ReferencedPakPureChecksums( void ) ) +DEF_PUBLIC( void fs_generate_reference_lists(void) ) +DEF_PUBLIC( void fs_update_client_download_map(clientProfile_t client_profile, FS_DownloadMapShared *&download_map) ) +DEF_PUBLIC( void fs_free_client_download_map(FS_DownloadMapShared *&download_map) ) +DEF_PUBLIC( fileHandle_t fs_open_download_pak(FS_DownloadMapShared *download_map, const char *path, unsigned int *size_out) ) + +/* ******************************************************************************** */ +// Misc +/* ******************************************************************************** */ + +// Indented debug prints +DEF_LOCAL( void fs_debug_indent_start(void) ) +DEF_LOCAL( void fs_debug_indent_stop(void) ) +DEF_LOCAL( void QDECL FS_DPrintf(const char *fmt, ...) __attribute__ ((format (printf, 1, 2))) ) + +// Hash Table +DEF_LOCAL( void fs_hashtable_initialize(fs_hashtable_t *hashtable, int bucket_count) ) +DEF_LOCAL( void fs_hashtable_insert(fs_hashtable_t *hashtable, fs_hashtable_entry_t *entry, unsigned int hash) ) +DEF_LOCAL( fs_hashtable_iterator_t fs_hashtable_iterate(fs_hashtable_t *hashtable, unsigned int hash, qboolean iterate_all) ) +DEF_LOCAL( void *fs_hashtable_next(fs_hashtable_iterator_t *iterator) ) +DEF_LOCAL( void fs_hashtable_free(fs_hashtable_t *hashtable, void (*free_entry)(fs_hashtable_entry_t *entry)) ) +DEF_LOCAL( void fs_hashtable_reset(fs_hashtable_t *hashtable, void (*free_entry)(fs_hashtable_entry_t *entry)) ) + +// Pk3 List +DEF_LOCAL( void pk3_list_initialize(pk3_list_t *pk3_list, unsigned int bucket_count) ) +DEF_LOCAL( int pk3_list_lookup(const pk3_list_t *pk3_list, unsigned int hash) ) +DEF_LOCAL( void pk3_list_insert(pk3_list_t *pk3_list, unsigned int hash) ) +DEF_LOCAL( void pk3_list_free(pk3_list_t *pk3_list) ) + +// File helper functions +#ifdef FSLOCAL +#define FS_FILE_BUFFER_SIZE 512 +#endif + +DEF_PUBLIC( const char *fs_file_extension(const fsc_file_t *file) ) +DEF_PUBLIC( qboolean fs_files_from_same_pk3(const fsc_file_t *file1, const fsc_file_t *file2) ) +DEF_LOCAL( int fs_get_source_dir_id(const fsc_file_t *file) ) +DEF_LOCAL( const char *fs_get_source_dir_string(const fsc_file_t *file) ) +DEF_LOCAL( void fs_file_to_stream(const fsc_file_t *file, fsc_stream_t *stream, qboolean include_source_dir, + qboolean include_mod, qboolean include_pk3_origin, qboolean include_size) ) +DEF_LOCAL( void fs_file_to_buffer(const fsc_file_t *file, char *buffer, unsigned int buffer_size, qboolean include_source_dir, + qboolean include_mod, qboolean include_pk3_origin, qboolean include_size) ) +DEF_PUBLIC( void fs_print_file_location(const fsc_file_t *file) ) + +// File disabled check function +#ifdef FSLOCAL +#define FD_CHECK_FILE_ENABLED 1 // Check if file is disabled in index +#define FD_CHECK_PURE_LIST 2 // Check if file is blocked by connected server pure configuration +#define FD_CHECK_READ_INACTIVE_MODS 4 // Check if file is blocked for file lookup by fs_read_inactive_mods setting +#define FD_CHECK_LIST_INACTIVE_MODS 8 // Check if file is blocked for file listing by fs_list_inactive_mods setting +#define FD_CHECK_LIST_AUXILIARY_SOURCEDIR 16 // Check if file is blocked for file listing due to auxiliary sourcedir +DEF_LOCAL( int fs_file_disabled(const fsc_file_t *file, int checks) ) +#endif + +// File Sorting Functions +DEF_LOCAL( void fs_write_sort_string(const char *string, fsc_stream_t *output, qboolean prioritize_shorter) ) +DEF_LOCAL( void fs_write_sort_filename(const fsc_file_t *file, fsc_stream_t *output) ) +DEF_LOCAL( void fs_write_sort_value(unsigned int value, fsc_stream_t *output) ) +DEF_LOCAL( void fs_generate_core_sort_key(const fsc_file_t *file, fsc_stream_t *output, qboolean use_server_pure_list) ) +DEF_LOCAL( int fs_compare_file(const fsc_file_t *file1, const fsc_file_t *file2, qboolean use_server_pure_list) ) +DEF_LOCAL( int fs_compare_pk3_source(const fsc_file_t *file1, const fsc_file_t *file2) ) + +// Misc Functions +DEF_PUBLIC( void fs_execute_config_file(const char *name, fs_config_type_t config_type, cbufExec_t exec_type, qboolean quiet) ) +DEF_PUBLIC( void *fs_load_game_dll(const fsc_file_t *dll_file, intptr_t (QDECL **entryPoint)(int, ...), + intptr_t (QDECL *systemcalls)(intptr_t, ...)) ) +DEF_PUBLIC( void FS_GetModDescription(const char *modDir, char *description, int descriptionLen) ) +DEF_PUBLIC( void FS_FilenameCompletion( const char *dir, const char *ext, + qboolean stripExt, void(*callback)(const char *s), qboolean allowNonPureFilesOnDisk ) ) +DEF_PUBLIC( qboolean FS_FilenameCompare( const char *s1, const char *s2 ) ) +DEF_PUBLIC( void QDECL FS_Printf( fileHandle_t f, const char *fmt, ... ) __attribute__ ((format (printf, 2, 3))) ) +DEF_LOCAL( void fs_comma_separated_list(const char **strings, int count, fsc_stream_t *output) ) +DEF_LOCAL( qboolean FS_idPak(const char *pak, const char *base, int numPaks) ) +DEF_LOCAL( void fs_sanitize_mod_dir(const char *source, char *target) ) +DEF_LOCAL( const char *fs_profile_to_string(FS_Profile profile) ) + +// QVM Hash Verification +DEF_LOCAL( qboolean calculate_file_sha256(const fsc_file_t *file, unsigned char *output) ) +DEF_LOCAL( qboolean fs_check_trusted_vm_file(const fsc_file_t *file) ) +DEF_LOCAL( void sha256_to_stream(unsigned char *sha, fsc_stream_t *output) ) + +// Core Pak Verification +DEF_LOCAL( void fs_check_core_paks(void) ) + +/* ******************************************************************************** */ +// Trusted VMs +/* ******************************************************************************** */ + +DEF_LOCAL( qboolean fs_check_trusted_vm_hash(unsigned char *hash) ) + +/* ******************************************************************************** */ +// Tremulous +/* ******************************************************************************** */ + +// Core Resource Precedence +DEF_LOCAL( int core_pk3_position(unsigned int hash) ) +DEF_LOCAL( int core_special_position(const char *special_id) ) + +// Base Directory & Mod Precedence +#ifdef FSLOCAL +enum FS_ModType { + MODTYPE_INACTIVE, + MODTYPE_BASE1, + MODTYPE_BASE2, + MODTYPE_BASE3, + MODTYPE_FS_BASEGAME, + MODTYPE_OVERRIDE_DIRECTORY, + MODTYPE_CURRENT_MOD +}; +#endif +DEF_LOCAL( FS_ModType fs_get_mod_type(const char *mod_dir, bool prioritize_fs_basegame=false) ) + +// System file manager functions +DEF_PUBLIC( bool FS_BrowseHomepath( void ) ) +DEF_PUBLIC( bool FS_OpenBaseGamePath( const char *baseGamePath ) ) +DEF_PUBLIC( bool FS_OpenModPath( const char *modPath ) ) + +#undef qboolean diff --git a/src/game/bg_misc.c b/src/game/bg_misc.c index ae3eb2200..2fd6e09d8 100644 --- a/src/game/bg_misc.c +++ b/src/game/bg_misc.c @@ -4004,7 +4004,7 @@ void BG_Bucket_Remove_Item_From_Bucket( } void* BG_Bucket_Select_A_Random_Item(unsigned int bucket_handle) { - Q_Bucket_Select_A_Random_Item(bucket_handle); + return Q_Bucket_Select_A_Random_Item(bucket_handle); } void BG_Bucket_Select_A_Specific_Item(unsigned int bucket_handle, void* item) { diff --git a/src/qcommon/cm_load.cpp b/src/qcommon/cm_load.cpp index 0ce0606a6..9796722e7 100644 --- a/src/qcommon/cm_load.cpp +++ b/src/qcommon/cm_load.cpp @@ -734,6 +734,16 @@ void CM_LoadMap( const char *name, bool clientload, int *checksum ) { #endif Com_DPrintf( "CM_LoadMap( %s, %i )\n", name, clientload ); +#ifdef NEW_FILESYSTEM + // Register the map here because, in the case of starting a local game through SV_Map_f + // the current map has not been set yet and we don't want the bsp lookup to be + // affected by a preexisting current map setting + fs_register_current_map(name); + + // Warning: Check below may be problematic, because sometimes different maps can be loaded + // with same name, due to pure lists, mod dirs, etc. +#endif + if ( !strcmp( cm.name, name ) && clientload ) { *checksum = last_checksum; return; diff --git a/src/qcommon/cmd.cpp b/src/qcommon/cmd.cpp index eaa341827..cdc50ef26 100644 --- a/src/qcommon/cmd.cpp +++ b/src/qcommon/cmd.cpp @@ -306,10 +306,12 @@ Cmd_Exec_f */ void Cmd_Exec_f( void ) { bool quiet; +#ifndef NEW_FILESYSTEM union { char *c; void *v; } f; +#endif char filename[MAX_QPATH]; quiet = !Q_stricmp(Cmd_Argv(0), "execq"); @@ -322,6 +324,9 @@ void Cmd_Exec_f( void ) { Q_strncpyz( filename, Cmd_Argv(1), sizeof( filename ) ); COM_DefaultExtension( filename, sizeof( filename ), ".cfg" ); +#ifdef NEW_FILESYSTEM + fs_execute_config_file(filename, FS_CONFIGTYPE_NONE, EXEC_INSERT, quiet); +#else FS_ReadFile( filename, &f.v); if (!f.c) { Com_Printf ("couldn't exec %s\n", filename); @@ -333,6 +338,7 @@ void Cmd_Exec_f( void ) { Cbuf_InsertText (f.c); FS_FreeFile (f.v); +#endif } diff --git a/src/qcommon/common.cpp b/src/qcommon/common.cpp index dcafcb85d..673f7ad21 100644 --- a/src/qcommon/common.cpp +++ b/src/qcommon/common.cpp @@ -486,7 +486,11 @@ void Engine_Exit(const char* p ) CL_Shutdown(p[0] ? p : "Client quit", true, true); VM_Forced_Unload_Done(); Com_Shutdown(); +#ifdef NEW_FILESYSTEM + fs_close_all_handles(); +#else FS_Shutdown(true); +#endif } Sys_Quit (); } @@ -1680,12 +1684,14 @@ void Com_InitHunkMemory( void ) int nMinAlloc; const char *pMsg = NULL; +#ifndef NEW_FILESYSTEM // make sure the file system has allocated and "not" freed any temp blocks // this allows the config and product id files ( journal files too ) to be loaded // by the file system without redunant routines in the file system utilizing different // memory systems if (FS_LoadStack() != 0) Com_Error( ERR_FATAL, "Hunk initialization failed. File system load stack not zero"); +#endif // allocate the stack based hunk allocator cv = Cvar_Get( "com_hunkMegs", DEF_COMHUNKMEGS_S, CVAR_LATCH | CVAR_ARCHIVE ); @@ -2481,6 +2487,15 @@ For controlling environment variables void Com_ExecuteCfg(void) { +#ifdef NEW_FILESYSTEM + fs_execute_config_file("default.cfg", FS_CONFIGTYPE_DEFAULT, EXEC_APPEND, qfalse); + if(!Com_SafeMode()) { + // skip the q3config.cfg and autoexec.cfg if "safe" is on the command line + fs_execute_config_file(Q3CONFIG_CFG, FS_CONFIGTYPE_SETTINGS, EXEC_APPEND, qfalse); + fs_execute_config_file("autoexec.cfg", FS_CONFIGTYPE_SETTINGS, EXEC_APPEND, qfalse); } + Cbuf_Execute(); + Com_Printf("\n"); +#else Cbuf_ExecuteText(EXEC_NOW, "exec default.cfg\n"); Cbuf_Execute(); // Always execute after exec to prevent text buffer overflowing @@ -2492,6 +2507,7 @@ void Com_ExecuteCfg(void) Cbuf_ExecuteText(EXEC_NOW, "exec autoexec.cfg\n"); Cbuf_Execute(); } +#endif } /* @@ -2524,7 +2540,11 @@ void Com_GameRestart(int checksumFeed, bool disconnect) CL_Shutdown("Game directory changed", disconnect, false); } +#ifdef NEW_FILESYSTEM + fs_set_mod_dir(Cvar_VariableString("fs_game"), qtrue); +#else FS_Restart(checksumFeed); +#endif // Clean out any user and VM created cvars Cvar_Restart(true); @@ -2558,6 +2578,16 @@ Expose possibility to change current running mod to the user void Com_GameRestart_f(void) { +#ifdef NEW_FILESYSTEM + // This will get sanitized in fs_set_mod_dir during the restart + Cvar_Set("fs_game", Cmd_Argv(1)); + +#ifndef DEDICATED + // Make sure the specified fs_game doesn't get overriden by cl_oldGame + extern bool cl_oldGameSet; + cl_oldGameSet = false; +#endif +#else if(!FS_FilenameCompare(Cmd_Argv(1), BASEGAME)) { // This is the standard base game. Servers and clients should @@ -2568,6 +2598,7 @@ void Com_GameRestart_f(void) } else Cvar_Set("fs_game", Cmd_Argv(1)); +#endif Com_GameRestart(0, true); } @@ -2692,7 +2723,11 @@ void Com_Init( char *commandLine ) com_homepath = Cvar_Get("com_homepath", "", CVAR_INIT); +#ifdef NEW_FILESYSTEM + fs_startup(); +#else FS_InitFilesystem (); +#endif Com_InitJournaling(); @@ -2715,6 +2750,10 @@ void Com_Init( char *commandLine ) // override anything from the config files with command line args Com_StartupVariable( NULL ); +#ifdef NEW_FILESYSTEM + fs_cache_initialize(); +#endif + // get dedicated here for proper hunk megs initialization #ifdef DEDICATED com_dedicated = Cvar_Get ("dedicated", "1", CVAR_INIT); @@ -2766,6 +2805,11 @@ void Com_Init( char *commandLine ) Cvar_Get ("protocol", va("%i", PROTOCOL_VERSION), CVAR_SERVERINFO | CVAR_ROM); com_gamename = Cvar_Get("com_gamename", GAMENAME_FOR_MASTER, CVAR_SERVERINFO | CVAR_INIT); +#ifdef NEW_FILESYSTEM + // Pid file init goes here if pid file support was present + // Sys_InitPIDFile( fs_pid_file_directory() ); +#endif + Sys_Init(); // Pick a random port value @@ -2874,7 +2918,14 @@ void Com_WriteConfigToFile( const char *filename ) { fileHandle_t f; +#ifdef NEW_FILESYSTEM + if(!Q_stricmp(filename, Q3CONFIG_CFG)) { + f = fs_open_settings_file_write(Q3CONFIG_CFG); } + else { + f = FS_FOpenFileWrite( filename ); } +#else f = FS_FOpenFileWrite( filename ); +#endif if ( !f ) { Com_Printf("Couldn't write %s.\n", filename ); @@ -3884,7 +3935,7 @@ void Com_Bucket_Remove_Item_From_Bucket( } void* Com_Bucket_Select_A_Random_Item(unsigned int bucket_handle) { - Q_Bucket_Select_A_Random_Item(bucket_handle); + return Q_Bucket_Select_A_Random_Item(bucket_handle); } void Com_Bucket_Select_A_Specific_Item(unsigned int bucket_handle, void* item) { diff --git a/src/qcommon/files.cpp b/src/qcommon/files.cpp index 68b72b30b..bc557186c 100644 --- a/src/qcommon/files.cpp +++ b/src/qcommon/files.cpp @@ -21,6 +21,7 @@ */ +#ifndef NEW_FILESYSTEM #include "files.h" #ifdef _WIN32 @@ -465,6 +466,33 @@ bool FS_OpenBaseGamePath( const char *baseGamePath ) return false; } +/* +============ +FS_OpenModPath + +Opens the given path for the +mod in the default file manager +============ +*/ +bool FS_OpenModPath( const char *modPath ) +{ + const char *homePath = Sys_DefaultHomePath( ); + const char *path; + + if (!homePath || !homePath[0]) + { + homePath = fs_basepath->string; + } + + path = FS_BuildOSPath( homePath, modPath, ""); + + if( FS_OpenWithDefault( path ) ) + return true; + + Com_Printf( S_COLOR_RED "FS_OpenModPath: failed to open the mod folder %s with the default file manager.\n" S_COLOR_WHITE, modPath ); + return false; +} + /* ============ FS_CreatePath @@ -3984,3 +4012,4 @@ const char *FS_GetCurrentGameDir(void) if (fs_gamedirvar->string[0]) return fs_gamedirvar->string; return BASEGAME; } +#endif diff --git a/src/qcommon/files.h b/src/qcommon/files.h index d54bdf485..490301d1b 100644 --- a/src/qcommon/files.h +++ b/src/qcommon/files.h @@ -20,6 +20,9 @@ #ifndef QC_FILES_H #define QC_FILES_H +#ifdef NEW_FILESYSTEM +#include "../filesystem/fspublic.h" +#else #ifdef __cplusplus extern "C" { #endif @@ -256,6 +259,7 @@ void FS_HomeRemove (const char* homePath); void FS_Remove (const char* osPath); bool FS_BrowseHomepath ( void ); bool FS_OpenBaseGamePath( const char *baseGamePath ); +bool FS_OpenModPath( const char *modPath ); bool FS_CreatePath (const char* OSPath); char* FS_BuildOSPath (const char* base, const char* game, const char* qpath); long FS_filelength (fileHandle_t f); @@ -282,5 +286,6 @@ extern char lastValidBase[MAX_OSPATH]; #ifdef __cplusplus } #endif +#endif #endif diff --git a/src/qcommon/q_platform.h b/src/qcommon/q_platform.h index b72d285dc..ccab51d08 100644 --- a/src/qcommon/q_platform.h +++ b/src/qcommon/q_platform.h @@ -105,8 +105,15 @@ extern "C" { #if defined( __WIN64__ ) #define ARCH_STRING "x86_64" + #define READABLE_ARCH_STRING "64" + #elif defined( __WIN32__ ) + #define ARCH_STRING "x86" + #define READABLE_ARCH_STRING "32" #elif defined _M_ALPHA #define ARCH_STRING "AXP" + #define READABLE_ARCH_STRING ARCH_STRING + #else + #define READABLE_ARCH_STRING ARCH_STRING #endif #define Q3_LITTLE_ENDIAN @@ -115,8 +122,8 @@ extern "C" { #define EXE_EXT ".exe" // For cl_updates.cpp - #define RELEASE_PACKAGE_NAME ( "release-mingw32-" ARCH_STRING ".zip" ) - #define RELEASE_SIGNATURE_NAME ( "release-mingw32-" ARCH_STRING ".zip.sig" ) + #define RELEASE_PACKAGE_NAME ( "release-windows-" READABLE_ARCH_STRING ".zip" ) + #define RELEASE_SIGNATURE_NAME ( "release-windows-" READABLE_ARCH_STRING ".zip.sig" ) #define GRANGER_EXE ( "granger" EXE_EXT ) #elif defined(_WIN32) || defined(__WIN32__) @@ -138,8 +145,15 @@ extern "C" { #if defined( _M_IX86 ) || defined( __i386__ ) #define ARCH_STRING "x86" + #define READABLE_ARCH_STRING "32" + #elif defined _M_X64 + #define ARCH_STRING "x86_64" + #define READABLE_ARCH_STRING "64" #elif defined _M_ALPHA #define ARCH_STRING "AXP" + #define READABLE_ARCH_STRING ARCH_STRING + #else + #define READABLE_ARCH_STRING ARCH_STRING #endif #define Q3_LITTLE_ENDIAN @@ -148,8 +162,8 @@ extern "C" { #define EXE_EXT ".exe" // For cl_updates.cpp - #define RELEASE_PACKAGE_NAME ( "release-mingw32-" ARCH_STRING ".zip" ) - #define RELEASE_SIGNATURE_NAME ( "release-mingw32-" ARCH_STRING ".zip.sig" ) + #define RELEASE_PACKAGE_NAME ( "release-windows-" READABLE_ARCH_STRING ".zip" ) + #define RELEASE_SIGNATURE_NAME ( "release-windows-" READABLE_ARCH_STRING ".zip.sig" ) #define GRANGER_EXE ( "granger" EXE_EXT ) #endif @@ -165,23 +179,28 @@ extern "C" { #ifdef __ppc__ #define ARCH_STRING "ppc" + #define READABLE_ARCH_STRING ARCH_STRING #define Q3_BIG_ENDIAN #elif defined __i386__ #define ARCH_STRING "x86" + #define READABLE_ARCH_STRING "32" #define Q3_LITTLE_ENDIAN #elif defined __x86_64__ #undef idx64 #define idx64 1 #define ARCH_STRING "x86_64" + #define READABLE_ARCH_STRING "64" #define Q3_LITTLE_ENDIAN + #else + #define READABLE_ARCH_STRING ARCH_STRING #endif #define DLL_EXT ".dylib" #define EXE_EXT // For cl_updates.cpp - #define RELEASE_PACKAGE_NAME ( "release-darwin-" ARCH_STRING ".zip" ) - #define RELEASE_SIGNATURE_NAME ( "release-darwin-" ARCH_STRING ".zip.sig" ) + #define RELEASE_PACKAGE_NAME ( "release-macos-" READABLE_ARCH_STRING ".zip" ) + #define RELEASE_SIGNATURE_NAME ( "release-macos-" READABLE_ARCH_STRING ".zip.sig" ) #define GRANGER_EXE ( "granger" EXE_EXT ) #endif @@ -206,12 +225,15 @@ extern "C" { #ifdef __ppc__ #define ARCH_STRING "ppc" + #define READABLE_ARCH_STRING ARCH_STRING #define Q3_BIG_ENDIAN #elif defined __i386__ # define ARCH_STRING "x86" + # define READABLE_ARCH_STRING "32" # define Q3_LITTLE_ENDIAN #elif defined __x86_64__ # define ARCH_STRING "x86_64" + # define READABLE_ARCH_STRING "64" # define Q3_LITTLE_ENDIAN #undef idx64 #define idx64 1 @@ -221,9 +243,11 @@ extern "C" { # endif # if defined __armhf__ # define ARCH_STRING "armhf" + #define READABLE_ARCH_STRING ARCH_STRING # define Q3_LITTLE_ENDIAN # else # define ARCH_STRING "armel" + #define READABLE_ARCH_STRING ARCH_STRING # define Q3_LITTLE_ENDIAN # endif #elif defined __aarch64__ @@ -231,15 +255,18 @@ extern "C" { # error "Big endian ARM is not supported" # endif # define ARCH_STRING "aarch64" + #define READABLE_ARCH_STRING ARCH_STRING # define Q3_LITTLE_ENDIAN + #else + #define READABLE_ARCH_STRING ARCH_STRING #endif #define DLL_EXT ".so" #define EXE_EXT // For cl_updates.cpp - #define RELEASE_PACKAGE_NAME ( "release-linux-" ARCH_STRING ".zip" ) - #define RELEASE_SIGNATURE_NAME ( "release-linux-" ARCH_STRING ".zip.sig" ) + #define RELEASE_PACKAGE_NAME ( "release-linux-" READABLE_ARCH_STRING ".zip" ) + #define RELEASE_SIGNATURE_NAME ( "release-linux-" READABLE_ARCH_STRING ".zip.sig" ) #define GRANGER_EXE ( "granger" EXE_EXT ) #endif diff --git a/src/qcommon/qcommon.h b/src/qcommon/qcommon.h index 9dff2a711..0be8fbd4b 100644 --- a/src/qcommon/qcommon.h +++ b/src/qcommon/qcommon.h @@ -164,6 +164,16 @@ typedef struct { void *evPtr; // this must be manually freed if not NULL } sysEvent_t; +#ifdef NEW_FILESYSTEM +// Different client configurations for multiprotocol servers (shared between server and filesystem) +typedef enum { + CLIENT_PROFILE_1_1, + CLIENT_PROFILE_GPP, + CLIENT_PROFILE_1_3, + CLIENT_PROFILE_COUNT +} clientProfile_t; +#endif + void Com_rgb_to_hsl(vec4_t rgb, vec4_t hsl); void Com_hsl_to_rgb(vec4_t hsl, vec4_t rgb); diff --git a/src/qcommon/vm.cpp b/src/qcommon/vm.cpp index ab36a3388..a18339111 100644 --- a/src/qcommon/vm.cpp +++ b/src/qcommon/vm.cpp @@ -79,7 +79,7 @@ VM_Init */ void VM_Init( void ) { Cvar_Get( "vm_cgame", "2", CVAR_ARCHIVE ); // !@# SHIP WITH SET TO 2 - Cvar_Get( "vm_game", "2", CVAR_ARCHIVE ); // !@# SHIP WITH SET TO 2 + Cvar_Get( "vm_game", "0", CVAR_ARCHIVE ); // !@# SHIP WITH SET TO 0 Cvar_Get( "vm_ui", "2", CVAR_ARCHIVE ); // !@# SHIP WITH SET TO 2 Cmd_AddCommand ("vmprofile", VM_VmProfile_f ); @@ -386,7 +386,11 @@ vmHeader_t *VM_LoadQVM( vm_t *vm, bool alloc, bool unpure) Com_sprintf( filename, sizeof(filename), "vm/%s.qvm", vm->name ); Com_Printf( "Loading vm file %s...\n", filename ); +#ifdef NEW_FILESYSTEM + header.v = fs_read_data(vm->source_file, 0, 0, "VM_LoadQVM"); +#else FS_ReadFileDir(filename, vm->searchPath, unpure, &header.v); +#endif if ( !header.h ) { Com_Printf( "Failed.\n" ); @@ -398,7 +402,11 @@ vmHeader_t *VM_LoadQVM( vm_t *vm, bool alloc, bool unpure) } // show where the qvm was loaded from +#ifdef NEW_FILESYSTEM + fs_print_file_location(vm->source_file); +#else FS_Which(filename, vm->searchPath); +#endif if( LittleLong( header.h->vmMagic ) == VM_MAGIC_VER2 ) { Com_Printf( "...which has vmMagic VM_MAGIC_VER2\n" ); @@ -543,6 +551,7 @@ vm_t *VM_Restart(vm_t *vm, bool unpure) { vmHeader_t *header; +#ifndef NEW_FILESYSTEM // DLL's can't be restarted in place if ( vm->dllHandle ) { char name[MAX_QPATH]; @@ -556,10 +565,20 @@ vm_t *VM_Restart(vm_t *vm, bool unpure) vm = VM_Create( name, systemCall, VMI_NATIVE ); return vm; } +#endif // load the image Com_Printf("VM_Restart()\n"); +#ifdef NEW_FILESYSTEM + if ( vm->dllHandle ) { + Sys_UnloadDll( vm->dllHandle ); + vm->dllHandle = fs_load_game_dll(vm->source_file, &vm->entryPoint, VM_DllSyscall); + if(!vm->dllHandle) Com_Error(ERR_DROP, "VM_Restart on dll failed"); + return vm; + } +#endif + if(!(header = VM_LoadQVM(vm, false, unpure))) { Com_Error(ERR_DROP, "VM_Restart failed"); @@ -584,9 +603,15 @@ vm_t *VM_Create( const char *module, intptr_t (*systemCalls)(intptr_t *), vmInterpret_t interpret ) { vm_t *vm; vmHeader_t *header; +#ifdef NEW_FILESYSTEM + int i; + int remaining; + bool is_dll = false; +#else int i, remaining, retval; char filename[MAX_OSPATH]; void *startSearch = NULL; +#endif if ( !module || !module[0] || !systemCalls ) { Com_Error( ERR_FATAL, "VM_Create: bad parms" ); @@ -617,6 +642,19 @@ vm_t *VM_Create( const char *module, intptr_t (*systemCalls)(intptr_t *), Q_strncpyz(vm->name, module, sizeof(vm->name)); +#ifdef NEW_FILESYSTEM + vm->source_file = fs_vm_lookup(module, interpret == VMI_NATIVE ? qfalse : qtrue, qfalse, &is_dll); + if(!vm->source_file) return 0; + + if(is_dll) { + vm->dllHandle = fs_load_game_dll(vm->source_file, &vm->entryPoint, VM_DllSyscall); + if(!vm->dllHandle) return 0; + vm->systemCall = systemCalls; + return vm; } + + header = VM_LoadQVM(vm, qtrue, qfalse); + if(!header) return 0; +#else do { retval = FS_FindVM(&startSearch, filename, sizeof(filename), module, (interpret == VMI_NATIVE)); @@ -648,6 +686,7 @@ vm_t *VM_Create( const char *module, intptr_t (*systemCalls)(intptr_t *), if(retval < 0) return NULL; +#endif vm->systemCall = systemCalls; diff --git a/src/qcommon/vm_local.h b/src/qcommon/vm_local.h index b4ce8441b..e8b9bddf6 100644 --- a/src/qcommon/vm_local.h +++ b/src/qcommon/vm_local.h @@ -23,6 +23,9 @@ along with Tremulous; if not, see */ #include "q_shared.h" #include "qcommon.h" +#ifdef NEW_FILESYSTEM +#include "qcommon/files.h" +#endif // Max number of arguments to pass from engine to vm's vmMain function. // command number + 3 arguments @@ -152,7 +155,11 @@ struct vm_s { //------------------------------------ char name[MAX_QPATH]; +#ifdef NEW_FILESYSTEM + const fsc_file_t *source_file; // Can be either qvm or dll +#else void *searchPath; // hint for FS_ReadFileDir() +#endif // for dynamic linked modules void *dllHandle; diff --git a/src/renderercommon/tr_public.h b/src/renderercommon/tr_public.h index 9d926d0c0..ccf09feed 100644 --- a/src/renderercommon/tr_public.h +++ b/src/renderercommon/tr_public.h @@ -24,6 +24,12 @@ along with Tremulous; if not, see #ifndef __TR_PUBLIC_H #define __TR_PUBLIC_H +#ifdef NEW_FILESYSTEM +typedef struct fsc_file_s fsc_file_t; +typedef struct fsc_shader_s fsc_shader_t; +#define LOOKUPFLAG_ENABLE_DDS 1 // Enable dds format for image lookups. Must match value in fspublic.h! +#endif + #include "renderercommon/tr_types.h" #define REF_API_VERSION 8 @@ -182,6 +188,14 @@ typedef struct { void (*Sys_GLimpSafeInit)( void ); void (*Sys_GLimpInit)( void ); bool (*Sys_LowPhysicalMemory)( void ); +#ifdef NEW_FILESYSTEM + const fsc_file_t *(*fs_general_lookup)(const char *name, int lookup_flags, bool debug); + const fsc_file_t *(*fs_image_lookup)(const char *name, int lookup_flags, bool debug); + const fsc_shader_t *(*fs_shader_lookup)(const char *name, int lookup_flags, bool debug); + char *(*fs_read_shader)(const fsc_shader_t *shader); + const char *(*fs_file_extension)(const fsc_file_t *file); + bool (*fs_files_from_same_pk3)(const fsc_file_t *file1, const fsc_file_t *file2); +#endif } refimport_t; diff --git a/src/renderergl1/tr_image.cpp b/src/renderergl1/tr_image.cpp index 0863db7d3..4bc61c944 100644 --- a/src/renderergl1/tr_image.cpp +++ b/src/renderergl1/tr_image.cpp @@ -947,6 +947,36 @@ Loads any of the supported image types into a cannonical 32 bit format. ================= */ +#ifdef NEW_FILESYSTEM +void R_LoadImageNewFS( const char *name, byte **pic, int *width, int *height ) { + char localName[MAX_QPATH]; + const fsc_file_t *file; + const char *extension; + int i; + + // Default outputs + *pic = NULL; + *width = 0; + *height = 0; + + // Look up the file + COM_StripExtension(name, localName, MAX_QPATH); + file = ri.fs_image_lookup(localName, 0, false); + if(!file) return; + + // Get extension + extension = ri.fs_file_extension(file); + if(!extension) extension = ""; + + for(i=0; i= 0 ; lod--) { if(lod) @@ -82,6 +118,7 @@ qhandle_t R_RegisterMD3(const char *name, model_t *mod) ri.FS_ReadFile( namebuf, &buf.v ); if(!buf.u) continue; +#endif ident = LittleLong(* (unsigned *) buf.u); if (ident == MD3_IDENT) @@ -102,6 +139,7 @@ qhandle_t R_RegisterMD3(const char *name, model_t *mod) if(numLoaded) { +#ifndef NEW_FILESYSTEM // duplicate into higher lod spots that weren't // loaded, in case the user changes r_lodbias on the fly for(lod--; lod >= 0; lod--) @@ -110,6 +148,7 @@ qhandle_t R_RegisterMD3(const char *name, model_t *mod) mod->md3[lod] = mod->md3[lod + 1]; } +#endif return mod->index; } diff --git a/src/renderergl1/tr_shader.cpp b/src/renderergl1/tr_shader.cpp index db9f140cf..c9d29b427 100644 --- a/src/renderergl1/tr_shader.cpp +++ b/src/renderergl1/tr_shader.cpp @@ -2516,6 +2516,10 @@ shader_t *R_FindShader( const char *name, int lightmapIndex, bool mipRawImage ) char strippedName[MAX_QPATH]; int hash; char *shaderText; +#ifdef NEW_FILESYSTEM + const fsc_shader_t *fsShader; + char *shaderTextToFree = 0; +#endif image_t *image; shader_t *sh; @@ -2563,6 +2567,12 @@ shader_t *R_FindShader( const char *name, int lightmapIndex, bool mipRawImage ) // // attempt to define shader from an explicit parameter file // +#ifdef NEW_FILESYSTEM + if(tr.new_filesystem) { + fsShader = ri.fs_shader_lookup(strippedName, 0, qfalse); + shaderText = shaderTextToFree = fsShader ? ri.fs_read_shader(fsShader) : 0; } + else +#endif shaderText = FindShaderInShaderText( strippedName ); if ( shaderText ) { // enable this when building a pak file to get a global list @@ -2576,6 +2586,9 @@ shader_t *R_FindShader( const char *name, int lightmapIndex, bool mipRawImage ) shader.defaultShader = true; } sh = FinishShader(); +#ifdef NEW_FILESYSTEM + if(tr.new_filesystem) ri.Free(shaderTextToFree); +#endif return sh; } @@ -3152,6 +3165,9 @@ void R_InitShaders( void ) { CreateInternalShaders(); +#ifdef NEW_FILESYSTEM + if(!tr.new_filesystem) +#endif ScanAndLoadShaderFiles(); CreateExternalShaders(); diff --git a/src/renderergl2/tr_image.cpp b/src/renderergl2/tr_image.cpp index 5e29ff79a..e12bac3a4 100644 --- a/src/renderergl2/tr_image.cpp +++ b/src/renderergl2/tr_image.cpp @@ -2295,6 +2295,42 @@ Loads any of the supported image types into a cannonical 32 bit format. ================= */ +#ifdef NEW_FILESYSTEM +void R_LoadImageNewFS( const char *name, byte **pic, int *width, int *height, GLenum *picFormat, int *numMips ) { + char localName[MAX_QPATH]; + const fsc_file_t *file; + const char *extension; + int i; + + // Default outputs + *pic = NULL; + *width = 0; + *height = 0; + *picFormat = GL_RGBA8; + *numMips = 0; + + // Look up the file + COM_StripExtension(name, localName, MAX_QPATH); + file = ri.fs_image_lookup(localName, r_ext_compressed_textures->integer ? LOOKUPFLAG_ENABLE_DDS : 0, false); + if(!file) return; + + // Get extension + extension = ri.fs_file_extension(file); + if(!extension) extension = ""; + + if(!Q_stricmp(extension, "dds")) { + R_LoadDDS(va("%s.%s", localName, extension), pic, width, height, picFormat, numMips); + return; } + + for(i=0; i= 0 ; lod--) { if(lod) @@ -83,6 +119,7 @@ qhandle_t R_RegisterMD3(const char *name, model_t *mod) size = ri.FS_ReadFile( namebuf, &buf.v ); if(!buf.u) continue; +#endif ident = LittleLong(* (unsigned *) buf.u); if (ident == MD3_IDENT) @@ -103,6 +140,7 @@ qhandle_t R_RegisterMD3(const char *name, model_t *mod) if(numLoaded) { +#ifndef NEW_FILESYSTEM // duplicate into higher lod spots that weren't // loaded, in case the user changes r_lodbias on the fly for(lod--; lod >= 0; lod--) @@ -111,6 +149,7 @@ qhandle_t R_RegisterMD3(const char *name, model_t *mod) mod->mdv[lod] = mod->mdv[lod + 1]; } +#endif return mod->index; } diff --git a/src/renderergl2/tr_shader.cpp b/src/renderergl2/tr_shader.cpp index b3ebacf62..980b4907d 100644 --- a/src/renderergl2/tr_shader.cpp +++ b/src/renderergl2/tr_shader.cpp @@ -3237,6 +3237,10 @@ shader_t *R_FindShader( const char *name, int lightmapIndex, bool mipRawImage ) char strippedName[MAX_QPATH]; int hash; char *shaderText; +#ifdef NEW_FILESYSTEM + const fsc_shader_t *fsShader; + char *shaderTextToFree = 0; +#endif image_t *image; shader_t *sh; @@ -3278,6 +3282,12 @@ shader_t *R_FindShader( const char *name, int lightmapIndex, bool mipRawImage ) // // attempt to define shader from an explicit parameter file // +#ifdef NEW_FILESYSTEM + if(tr.new_filesystem) { + fsShader = ri.fs_shader_lookup(strippedName, r_ext_compressed_textures->integer ? LOOKUPFLAG_ENABLE_DDS : 0, qfalse); + shaderText = shaderTextToFree = fsShader ? ri.fs_read_shader(fsShader) : 0; } + else +#endif shaderText = FindShaderInShaderText( strippedName ); if ( shaderText ) { // enable this when building a pak file to get a global list @@ -3291,6 +3301,9 @@ shader_t *R_FindShader( const char *name, int lightmapIndex, bool mipRawImage ) shader.defaultShader = true; } sh = FinishShader(); +#ifdef NEW_FILESYSTEM + if(tr.new_filesystem) ri.Free(shaderTextToFree); +#endif return sh; } @@ -3885,6 +3898,9 @@ void R_InitShaders( void ) { CreateInternalShaders(); +#ifdef NEW_FILESYSTEM + if(!tr.new_filesystem) +#endif ScanAndLoadShaderFiles(); CreateExternalShaders(); diff --git a/src/sdl/sdl_glimp.cpp b/src/sdl/sdl_glimp.cpp index 9ea4034f4..51b7e7ea2 100644 --- a/src/sdl/sdl_glimp.cpp +++ b/src/sdl/sdl_glimp.cpp @@ -156,7 +156,7 @@ static void GLimp_DetectAvailableModes(void) return; } - modes = (SDL_Rect*)SDL_calloc( (size_t)numSDLModes, sizeof( SDL_Rect ) ); + modes = (SDL_Rect*)SDL_calloc( (size_t)numSDLModes + 1, sizeof( SDL_Rect ) ); if ( !modes ) { ri.Error( ERR_FATAL, "Out of memory" ); diff --git a/src/server/server.h b/src/server/server.h index 5144e3b82..321ed8936 100644 --- a/src/server/server.h +++ b/src/server/server.h @@ -101,6 +101,9 @@ struct server_t { int snapshotCounter; // incremented for each snapshot built int timeResidual; // <= 1000 / sv_frame->value int nextFrameTime; // when time > nextFrameTime, process world +#ifdef NEW_FILESYSTEM + char versionedConfigstrings[2][CLIENT_PROFILE_COUNT][BIG_INFO_STRING]; +#endif configString_t configstrings[MAX_CONFIGSTRINGS]; svEntity_t svEntities[MAX_GENTITIES]; @@ -152,6 +155,9 @@ struct client_t { clientState_t state; char userinfo[MAX_INFO_STRING]; // name, etc char userinfobuffer[MAX_INFO_STRING]; ///< used for buffering of user info +#ifdef NEW_FILESYSTEM + clientProfile_t clientProfile; // version profile for info strings, pak references, etc. +#endif char reliableCommands[MAX_RELIABLE_COMMANDS][MAX_STRING_CHARS]; int reliableSequence; // last added reliable message, not necesarily sent or acknowledged yet @@ -171,6 +177,9 @@ struct client_t { char name_ansi[MAX_COLORFUL_NAME_LENGTH]; //hex colors are aproximated to ansi for backwards compatibility // downloading +#ifdef NEW_FILESYSTEM + FS_DownloadMapShared *downloadMap; // used to match client download request to actual file +#endif char downloadName[MAX_QPATH]; // if not empty string, we are downloading fileHandle_t download; // file being downloaded int downloadSize; // total bytes (can't use EOF because of paks) @@ -195,8 +204,10 @@ struct client_t { int ping; int rate; // bytes / second int snapshotMsec; // requests a snapshot every snapshotMsec unless rate choked +#ifndef NEW_FILESYSTEM int pureAuthentic; bool gotCP; // TTimo - additional flag to distinguish between a bad pure checksum, and no cp command at all +#endif netchan_t netchan; // TTimo // queuing outgoing fragmented messages to send them properly, without udp packet bursts diff --git a/src/server/sv_ccmds.cpp b/src/server/sv_ccmds.cpp index 54c16d20f..e682fa8b4 100644 --- a/src/server/sv_ccmds.cpp +++ b/src/server/sv_ccmds.cpp @@ -58,7 +58,17 @@ static void SV_Map_f( void ) { // make sure the level exists before trying to change, so that // a typo at the server console won't end the game Com_sprintf (expanded, sizeof(expanded), "maps/%s.bsp", map); +#ifdef NEW_FILESYSTEM + // Refresh in case a map was just manually added + fs_refresh_auto(); + + // Perform the check without respect to current connected pure list + // If we go forward loading the map, the pure list will be cleared in SV_SpawnServer + // before further access to the bsp + if(!fs_general_lookup(expanded, LOOKUPFLAG_IGNORE_PURE_LIST|LOOKUPFLAG_IGNORE_CURRENT_MAP, qfalse)) { +#else if ( FS_ReadFile (expanded, NULL) == -1 ) { +#endif Com_Printf ("Can't find map %s\n", expanded); return; } diff --git a/src/server/sv_client.cpp b/src/server/sv_client.cpp index e76e77ad6..76580aa6e 100644 --- a/src/server/sv_client.cpp +++ b/src/server/sv_client.cpp @@ -467,6 +467,12 @@ void SV_DirectConnect( netadr_t from ) { Cvar_Set( va( "sv_clAltProto%i", clientNum ), ( version == 69 ? "2" : version == 70 ? "1" : "0" ) ); +#ifdef NEW_FILESYSTEM + // save the client version profile + newcl->clientProfile = ( version == 69 ? CLIENT_PROFILE_1_1 : version == 70 ? CLIENT_PROFILE_GPP : + CLIENT_PROFILE_1_3 ); +#endif + // save the challenge newcl->challenge = challenge; @@ -543,6 +549,10 @@ void SV_FreeClient(client_t *client) client->queuedVoipPackets = 0; #endif +#ifdef NEW_FILESYSTEM + fs_free_client_download_map(client->downloadMap); +#endif + SV_Netchan_FreeQueue(client); SV_CloseDownload(client); } @@ -607,7 +617,9 @@ void SV_DropClient( client_t *drop, const char *reason ) { } } +#ifndef NEW_FILESYSTEM extern char alternateInfos[2][2][BIG_INFO_STRING]; +#endif /* ================ @@ -630,8 +642,17 @@ static void SV_SendClientGameState( client_t *client ) { Com_DPrintf ("SV_SendClientGameState() for %s\n", client->name); Com_DPrintf( "Going from CS_CONNECTED to CS_PRIMED for %s\n", client->name ); client->state = CS_PRIMED; +#ifndef NEW_FILESYSTEM client->pureAuthentic = 0; client->gotCP = false; +#endif + +#ifdef NEW_FILESYSTEM + // Store download map for the client so all the pk3s in the gamestate will be available + // for the client to download even if the global download list changed during the client + // download sequence + fs_update_client_download_map(client->clientProfile, client->downloadMap); +#endif // when we receive the first packet from the client, we will // notice that it is from a different serverid and that the @@ -656,8 +677,13 @@ static void SV_SendClientGameState( client_t *client ) { // write the configstrings for ( start = 0 ; start < MAX_CONFIGSTRINGS ; start++ ) { +#ifdef NEW_FILESYSTEM + if ( start <= CS_SYSTEMINFO ) { + configstring = sv.versionedConfigstrings[start][client->clientProfile]; +#else if ( start <= CS_SYSTEMINFO && client->netchan.alternateProtocol != 0 ) { configstring = alternateInfos[start][ client->netchan.alternateProtocol - 1 ]; +#endif } else { configstring = sv.configstrings[start].s; } @@ -838,6 +864,36 @@ static void SV_BeginDownload_f( client_t *cl ) { Q_strncpyz( cl->downloadName, Cmd_Argv(1), sizeof(cl->downloadName) ); } +#ifdef NEW_FILESYSTEM +static void SV_OpenDownloadError(client_t *cl, msg_t *msg, const char *errorMessage) { + // Sends error message to client + MSG_WriteByte( msg, svc_download ); + MSG_WriteShort( msg, 0 ); // client is expecting block zero + MSG_WriteLong( msg, -1 ); // illegal file size + MSG_WriteString( msg, errorMessage ); } + +static qboolean SV_OpenDownload(client_t *cl, msg_t *msg) { + // Returns qtrue on success, qfalse otherwise + char pak_name[MAX_QPATH]; + COM_StripExtension(cl->downloadName, pak_name, sizeof(pak_name)); + + if(!(sv_allowDownload->integer & DLF_ENABLE) || (sv_allowDownload->integer & DLF_NO_UDP) ) { + Com_Printf("clientDownload: %d : \"%s\" UDP download disabled by server sv_allowDownload setting\n", + (int) (cl - svs.clients), cl->downloadName); + SV_OpenDownloadError(cl, msg, va("Could not download \"%s\" because autodownloading is disabled on the server.\n" + "Set autodownload to No in your settings and you might be able to join the game anyway.\n", cl->downloadName)); + return qfalse; } + + cl->download = fs_open_download_pak(cl->downloadMap, cl->downloadName, (unsigned int *)&cl->downloadSize); + if(!cl->download) { + Com_Printf("clientDownload: %d : \"%s\" failed to load download pk3\n", (int) (cl - svs.clients), cl->downloadName); + SV_OpenDownloadError(cl, msg, va("File \"%s\" not available on server for downloading.\n" + "Try reconnecting to the server.\n", cl->downloadName)); + return qfalse; } + + return qtrue; } +#endif + /* ================== SV_WriteDownloadToClient @@ -849,16 +905,23 @@ Fill up msg with data, return number of download blocks added int SV_WriteDownloadToClient(client_t *cl, msg_t *msg) { int curindex; +#ifndef NEW_FILESYSTEM int unreferenced = 1; char errorMessage[1024]; char pakbuf[MAX_QPATH], *pakptr; int numRefPaks; +#endif if (!*cl->downloadName) return 0; // Nothing being downloaded if(!cl->download) { +#ifdef NEW_FILESYSTEM + if(!SV_OpenDownload(cl, msg)) { + *cl->downloadName = 0; + return 1; } +#else // Chop off filename extension. Com_sprintf(pakbuf, sizeof(pakbuf), "%s", cl->downloadName); pakptr = strrchr(pakbuf, '.'); @@ -933,6 +996,7 @@ int SV_WriteDownloadToClient(client_t *cl, msg_t *msg) return 1; } +#endif Com_Printf( "clientDownload: %d : beginning \"%s\"\n", (int) (cl - svs.clients), cl->downloadName ); @@ -1099,6 +1163,20 @@ static void SV_Disconnect_f( client_t *cl ) { SV_DropClient( cl, "disconnected" ); } +#ifdef NEW_FILESYSTEM +/* +================= +SV_VerifyPaks_f / SV_ResetPureClient_f + +Placeholder functions to handle the deprecated cp and vdr commands +================= +*/ +static void SV_VerifyPaks_f( client_t *cl ) { +} + +static void SV_ResetPureClient_f( client_t *cl ) { +} +#else /* ================= SV_VerifyPaks_f @@ -1274,6 +1352,7 @@ static void SV_ResetPureClient_f( client_t *cl ) { cl->pureAuthentic = 0; cl->gotCP = false; } +#endif /* ================= @@ -1619,6 +1698,7 @@ static void SV_UserMove( client_t *cl, msg_t *msg, bool delta ) { // save time for ping calculation cl->frames[ cl->messageAcknowledge & PACKET_MASK ].messageAcked = svs.time; +#ifndef NEW_FILESYSTEM // TTimo // catch the no-cp-yet situation before SV_ClientEnterWorld // if CS_ACTIVE, then it's time to trigger a new gamestate emission @@ -1632,6 +1712,7 @@ static void SV_UserMove( client_t *cl, msg_t *msg, bool delta ) { } return; } +#endif // if this is the first usercmd we have received // this gamestate, put the client into the world @@ -1640,11 +1721,13 @@ static void SV_UserMove( client_t *cl, msg_t *msg, bool delta ) { // the moves can be processed normaly } +#ifndef NEW_FILESYSTEM // a bad cp command was sent, drop the client if (sv_pure->integer != 0 && cl->pureAuthentic == 0) { SV_DropClient( cl, "Cannot validate pure client!"); return; } +#endif if ( cl->state != CS_ACTIVE ) { cl->deltaMessage = -1; diff --git a/src/server/sv_game.cpp b/src/server/sv_game.cpp index 23e5212c2..ab353d812 100644 --- a/src/server/sv_game.cpp +++ b/src/server/sv_game.cpp @@ -325,14 +325,27 @@ intptr_t SV_GameSystemCalls( intptr_t *args ) { return 0; case G_FS_FOPEN_FILE: +#ifdef NEW_FILESYSTEM + return FS_FOpenFileByModeOwner( (const char*)VMA(1), (fileHandle_t*)VMA(2), (FS_Mode)args[3], FS_HANDLEOWNER_QAGAME ); +#else return FS_FOpenFileByMode( (const char*)VMA(1), (fileHandle_t*)VMA(2), (FS_Mode)args[3] ); +#endif case G_FS_READ: +#ifdef NEW_FILESYSTEM + if(fs_handle_get_owner(args[3]) != FS_HANDLEOWNER_QAGAME) return 0; +#endif FS_Read( VMA(1), args[2], args[3] ); return 0; case G_FS_WRITE: +#ifdef NEW_FILESYSTEM + if(fs_handle_get_owner(args[3]) != FS_HANDLEOWNER_QAGAME) return 0; +#endif FS_Write( VMA(1), args[2], args[3] ); return 0; case G_FS_FCLOSE_FILE: +#ifdef NEW_FILESYSTEM + if(fs_handle_get_owner(args[1]) != FS_HANDLEOWNER_QAGAME) return 0; +#endif FS_FCloseFile( args[1] ); return 0; case G_FS_GETFILELIST: @@ -340,6 +353,9 @@ intptr_t SV_GameSystemCalls( intptr_t *args ) { case G_FS_GETFILTEREDFILES: return FS_GetFilteredFiles( (const char*)VMA(1), (const char*)VMA(2), (char*)VMA(3), (char*)VMA(4), args[5] ); case G_FS_SEEK: +#ifdef NEW_FILESYSTEM + if(fs_handle_get_owner(args[1]) != FS_HANDLEOWNER_QAGAME) return 0; +#endif return FS_Seek( args[1], args[2], (FS_Origin)args[3] ); case G_LOCATE_GAME_DATA: SV_LocateGameData( (sharedEntity_t*)VMA(1), args[2], args[3], (playerState_t*)VMA(4), args[5] ); @@ -513,6 +529,9 @@ void SV_ShutdownGameProgs( void ) { return; } VM_Call( sv.gvm, GAME_SHUTDOWN, false ); +#ifdef NEW_FILESYSTEM + fs_close_owner_handles(FS_HANDLEOWNER_QAGAME); +#endif VM_Free( sv.gvm ); sv.gvm = NULL; } @@ -557,6 +576,9 @@ void SV_RestartGameProgs( void ) { return; } VM_Call( sv.gvm, GAME_SHUTDOWN, true ); +#ifdef NEW_FILESYSTEM + fs_close_owner_handles(FS_HANDLEOWNER_QAGAME); +#endif // do a restart instead of a free sv.gvm = VM_Restart(sv.gvm, true); diff --git a/src/server/sv_init.cpp b/src/server/sv_init.cpp index 1bdd16e27..846138d40 100644 --- a/src/server/sv_init.cpp +++ b/src/server/sv_init.cpp @@ -31,7 +31,9 @@ along with Tremulous; if not, see // we even log attacks when the server is waiting for rcon and doesn't run a map int attHandle = 0; // server attack log file handle +#ifndef NEW_FILESYSTEM char alternateInfos[2][2][BIG_INFO_STRING]; +#endif /* =============== @@ -55,10 +57,17 @@ static void SV_SendConfigstring(client_t *client, int i) return; } +#ifdef NEW_FILESYSTEM + if (i <= CS_SYSTEMINFO) + { + configstring = sv.versionedConfigstrings[i][client->clientProfile]; + } +#else if (i <= CS_SYSTEMINFO && client->netchan.alternateProtocol != 0) { configstring = alternateInfos[i][client->netchan.alternateProtocol - 1]; } +#endif else { configstring = sv.configstrings[i].s; @@ -136,7 +145,11 @@ SV_SetConfigstring */ void SV_SetConfigstring(int idx, const char *val) { +#ifdef NEW_FILESYSTEM + bool modified[CLIENT_PROFILE_COUNT] = {false}; +#else bool modified[3] = {false, false, false}; +#endif int i; client_t *client; @@ -152,6 +165,53 @@ void SV_SetConfigstring(int idx, const char *val) if (idx <= CS_SYSTEMINFO) { +#ifdef NEW_FILESYSTEM + for (i = 0; i < CLIENT_PROFILE_COUNT; ++i) + { + clientProfile_t profile = (clientProfile_t)i; + char info[BIG_INFO_STRING]; + Q_strncpyz(info, val, sizeof(info)); + + if (idx == CS_SERVERINFO) + { + if (profile == CLIENT_PROFILE_1_1) + Info_SetValueForKey_Big(info, "protocol", "69"); + if (profile == CLIENT_PROFILE_GPP) + Info_SetValueForKey_Big(info, "protocol", "70"); + } + + if (idx == CS_SYSTEMINFO) + { + // Obtain reference strings from cvars corresponding to version profile + FS_ReferenceProfile reference_profile(profile); + Info_SetValueForKey_Big(info, "sv_referencedPaks", + Cvar_VariableString(reference_profile.download_hashes_cvar.c_str())); + Info_SetValueForKey_Big(info, "sv_referencedPakNames", + Cvar_VariableString(reference_profile.download_names_cvar.c_str())); + Info_SetValueForKey_Big(info, "sv_paks", + Cvar_VariableString(reference_profile.pure_hashes_cvar.c_str())); + Info_SetValueForKey_Big(info, "sv_pakNames", + Cvar_VariableString(reference_profile.pure_names_cvar.c_str())); + + if (profile == CLIENT_PROFILE_1_1) + { + Info_SetValueForKey_Big(info, "cl_allowDownload", "1, you should set it yourself"); + if (!(sv_allowDownload->integer & DLF_NO_REDIRECT)) + { + Info_SetValueForKey_Big(info, "sv_wwwBaseURL", Cvar_VariableString("sv_dlUrl")); + Info_SetValueForKey_Big( + info, "sv_wwwDownload", Cvar_VariableString("1, you should set it yourself")); + } + } + } + + if (strcmp(info, sv.versionedConfigstrings[idx][profile])) + { + modified[profile] = true; + strcpy(sv.versionedConfigstrings[idx][profile], info); + } + } +#else for (i = 1; i < 3; ++i) { char info[BIG_INFO_STRING]; @@ -195,6 +255,7 @@ void SV_SetConfigstring(int idx, const char *val) { return; } +#endif } else { @@ -216,10 +277,17 @@ void SV_SetConfigstring(int idx, const char *val) // send the data to all relevent clients for (i = 0, client = svs.clients; i < sv_maxclients->integer; i++, client++) { +#ifdef NEW_FILESYSTEM + if (idx <= CS_SYSTEMINFO && !modified[client->clientProfile]) + { + continue; + } +#else if (idx <= CS_SYSTEMINFO && !modified[client->netchan.alternateProtocol]) { continue; } +#endif if (client->state < CS_ACTIVE) { @@ -517,10 +585,12 @@ static void SV_ClearServer(void) for (i = 0; i < MAX_CONFIGSTRINGS; i++) { +#ifndef NEW_FILESYSTEM if (i <= CS_SYSTEMINFO) { alternateInfos[i][0][0] = alternateInfos[i][1][0] = '\0'; } +#endif if (sv.configstrings[i].s) { Z_Free(sv.configstrings[i].s); @@ -529,6 +599,7 @@ static void SV_ClearServer(void) ::memset(&sv, 0, sizeof(sv)); } +#ifndef NEW_FILESYSTEM /* ================ SV_TouchCGame @@ -548,6 +619,7 @@ static void SV_TouchCGame(void) FS_FCloseFile(f); } } +#endif /* ================ @@ -563,7 +635,9 @@ void SV_SpawnServer(char *server) int i; int checksum; char systemInfo[16384]; +#ifndef NEW_FILESYSTEM const char *p; +#endif // shut down the existing game if it is running SV_ShutdownGameProgs(); @@ -571,6 +645,12 @@ void SV_SpawnServer(char *server) Com_Printf("------ Server Initialization ------\n"); Com_Printf("Server: %s\n", server); +#ifdef NEW_FILESYSTEM + // This is sometimes called in CL_MapLoading->CL_Disconnect, but not in the case + // of a previous local game, so do it here to be safe + fs_disconnect_cleanup(); +#endif + // if not running a dedicated server CL_MapLoading will connect the client to the server // also print some status stuff CL_MapLoading(); @@ -622,10 +702,12 @@ void SV_SpawnServer(char *server) SV_ClearServer(); for (i = 0; i < MAX_CONFIGSTRINGS; i++) { +#ifndef NEW_FILESYSTEM if (i <= CS_SYSTEMINFO) { alternateInfos[i][0][0] = alternateInfos[i][1][0] = '\0'; } +#endif sv.configstrings[i].s = CopyString(""); sv.configstrings[i].restricted = false; ::memset(&sv.configstrings[i].clientList, 0, sizeof(clientList_t)); @@ -636,7 +718,9 @@ void SV_SpawnServer(char *server) // get a new checksum feed and restart the file system sv.checksumFeed = (((int)rand() << 16) ^ rand()) ^ Com_Milliseconds(); +#ifndef NEW_FILESYSTEM FS_Restart(sv.checksumFeed); +#endif // advertise GPP-compatible extensions Cvar_Set("sv_gppExtension", "1"); @@ -706,6 +790,10 @@ void SV_SpawnServer(char *server) sv.time += 100; svs.time += 100; +#ifdef NEW_FILESYSTEM + // Set download and pure list cvars + fs_generate_reference_lists(); +#else if (sv_pure->integer) { // the server sends these to the clients so they will only @@ -747,6 +835,7 @@ void SV_SpawnServer(char *server) Cvar_Set("sv_referencedPakNames", p); p = FS_ReferencedPakNames(true); Cvar_Set("sv_referencedAlternatePakNames", p); +#endif // save systeminfo and serverinfo strings Q_strncpyz(systemInfo, Cvar_InfoString_Big(CVAR_SYSTEMINFO), sizeof(systemInfo)); @@ -880,6 +969,7 @@ void SV_Init(void) Cvar_CheckRange(sv_voip, 0, 1, true); sv_voipProtocol = Cvar_Get("sv_voipProtocol", sv_voip->integer ? "opus" : "", CVAR_SYSTEMINFO | CVAR_ROM); #endif +#ifndef NEW_FILESYSTEM Cvar_Get("sv_paks", "", CVAR_SYSTEMINFO | CVAR_ROM); Cvar_Get("sv_pakNames", "", CVAR_SYSTEMINFO | CVAR_ROM); Cvar_Get("sv_referencedPaks", "", CVAR_SYSTEMINFO | CVAR_ROM); @@ -888,6 +978,7 @@ void SV_Init(void) Cvar_Get("sv_alternatePakNames", "", CVAR_ALTERNATE_SYSTEMINFO | CVAR_ROM); Cvar_Get("sv_referencedAlternatePaks", "", CVAR_ALTERNATE_SYSTEMINFO | CVAR_ROM); Cvar_Get("sv_referencedAlternatePakNames", "", CVAR_ALTERNATE_SYSTEMINFO | CVAR_ROM); +#endif // server vars sv_rconPassword = Cvar_Get("rconPassword", "", CVAR_TEMP); diff --git a/src/sys/sys_main.cpp b/src/sys/sys_main.cpp index d01782d5a..fcab5a3d6 100644 --- a/src/sys/sys_main.cpp +++ b/src/sys/sys_main.cpp @@ -191,6 +191,12 @@ Sys_PIDFileName */ static std::string Sys_PIDFileName( void ) { +#ifdef NEW_FILESYSTEM + char buffer[FS_MAX_PATH]; + if(fs_generate_path_writedir(PID_FILENAME, 0, 0, 0, + buffer, sizeof(buffer))) return buffer; + return ""; +#else const char *homePath = Cvar_VariableString( "fs_homepath" ); std::string pidfile; @@ -202,6 +208,7 @@ static std::string Sys_PIDFileName( void ) } return pidfile; +#endif } /* @@ -489,6 +496,37 @@ from executable path, then fs_basepath. */ void *Sys_LoadDll(const char *name, bool useSystemLib) { +#ifdef NEW_FILESYSTEM + void *dllhandle = 0; + char path[FS_MAX_PATH]; + + // Extra sanity check - this check shouldn't be security critical because it shouldn't be possible + // to get an arbitrary file written to the dll search locations, regardless of extension + if(!Sys_DllExtension(name)) { + Com_Printf("Refusing to attempt to load library \"%s\": Extension not allowed.\n", name); + return NULL; } + + if(useSystemLib) { + Com_Printf("Trying to load \"%s\"...\n", name); + if(fs_generate_path(name, 0, 0, FS_ALLOW_DLL, 0, 0, path, sizeof(path))) + dllhandle = Sys_LoadLibrary(path); } + + if(!dllhandle) { + const char *topDir = Sys_BinaryPath(); + if(!*topDir) topDir = "."; + Com_Printf("Trying to load \"%s\" from \"%s\"...\n", name, topDir); + if(fs_generate_path(topDir, name, 0, FS_NO_SANITIZE, FS_ALLOW_DLL, 0, path, sizeof(path))) + dllhandle = Sys_LoadLibrary(path); } + + if(!dllhandle) { + const char *basePath = Cvar_VariableString("fs_basepath"); + if(!basePath || !*basePath) basePath = "."; + Com_Printf("Trying to load \"%s\" from \"%s\"...\n", name, basePath); + if(fs_generate_path(basePath, name, 0, FS_NO_SANITIZE, FS_ALLOW_DLL, 0, path, sizeof(path))) + dllhandle = Sys_LoadLibrary(path); } + + if(!dllhandle) Com_Printf("Loading \"%s\" failed\n", name); +#else void *dllhandle; if (!Sys_DllExtension(name)) @@ -549,6 +587,7 @@ void *Sys_LoadDll(const char *name, bool useSystemLib) Com_Printf("Loading \"%s\" failed\n", name); } } +#endif return dllhandle; } diff --git a/src/sys/sys_unix.cpp b/src/sys/sys_unix.cpp index e55655c39..6e321ea55 100644 --- a/src/sys/sys_unix.cpp +++ b/src/sys/sys_unix.cpp @@ -244,7 +244,11 @@ FILE *Sys_Mkfifo( const char *ospath ) // if file already exists AND is a pipefile, remove it if( !stat( ospath, &buf ) && S_ISFIFO( buf.st_mode ) ) +#ifdef NEW_FILESYSTEM + remove(ospath); +#else FS_Remove( ospath ); +#endif result = mkfifo( ospath, 0600 ); if( result != 0 ) @@ -594,11 +598,18 @@ void Sys_ErrorDialog( const char *error ) char buffer[ 1024 ]; unsigned int size; int f = -1; +#ifdef NEW_FILESYSTEM + char ospath[FS_MAX_PATH]; + char *fileName = ospath; + + fs_generate_path_writedir("crashlog.txt", 0, 0, 0, ospath, sizeof(ospath)); +#else const char *homepath = Cvar_VariableString( "fs_homepath" ); const char *gamedir = Cvar_VariableString( "fs_game" ); const char *fileName = "crashlog.txt"; char *dirpath = FS_BuildOSPath( homepath, gamedir, ""); char *ospath = FS_BuildOSPath( homepath, gamedir, fileName ); +#endif Sys_Print( va( "%s\n", error ) ); @@ -606,6 +617,7 @@ void Sys_ErrorDialog( const char *error ) Sys_Dialog( DT_ERROR, va( "%s. See \"%s\" for details.", error, ospath ), "Error" ); #endif +#ifndef NEW_FILESYSTEM // Make sure the write path for the crashlog exists... if(!Sys_Mkdir(homepath)) @@ -619,6 +631,7 @@ void Sys_ErrorDialog( const char *error ) Com_Printf("ERROR: couldn't create path '%s' for crash log.\n", dirpath); return; } +#endif // We might be crashing because we maxed out the Quake MAX_FILE_HANDLES, // which will come through here, so we don't want to recurse forever by diff --git a/src/ui/ui_shared.c b/src/ui/ui_shared.c index a4a7de935..4efb9adf0 100644 --- a/src/ui/ui_shared.c +++ b/src/ui/ui_shared.c @@ -1943,7 +1943,7 @@ void Script_playLoopedBucket(itemDef_t *item, char **args) DC->setCVar(selected_list_cvar, selected_list_cvar_string); } - if(val) { + if(val[0] != '\0') { strcpy(play_looped_sound_path, va("%s/%s", sound_bucket_directory, val)); DC->stopBackgroundTrack(); DC->startBackgroundTrack(play_looped_sound_path, play_looped_sound_path);