admin/x264
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

x264 source for verification 2026-05-22

Browse Source
This commit is contained in:
admin
2026-05-22 16:45:04 +08:00
commit 4647f166e5
270 changed files with 166522 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

15
tools/bash-autocomplete.sh Normal file
View File

@@ -0,0 +1,15 @@
_x264()
{
local path args cur prev
path="${COMP_LINE%%[[:blank:]]*}"
args="${COMP_LINE:${#path}:$((COMP_POINT-${#path}))}"
cur="${args##*[[:blank:]=]}"
prev="$(sed 's/[[:blank:]=]*$//; s/^.*[[:blank:]]//' <<< "${args%%"$cur"}")"
# Expand ~
printf -v path '%q' "$path" && eval path="${path/#'\~'/'~'}"
COMPREPLY=($("$path" --autocomplete "$prev" "$cur")) && compopt +o default
} 2>/dev/null
complete -o default -F _x264 x264

235
tools/checkasm-a.asm Normal file
View File

@@ -0,0 +1,235 @@
;*****************************************************************************
;* checkasm-a.asm: assembly check tool
;*****************************************************************************
;* Copyright (C) 2008-2025 x264 project
;*
;* Authors: Loren Merritt <lorenm@u.washington.edu>
;* Henrik Gramner <henrik@gramner.com>
;*
;* This program is free software; you can redistribute it and/or modify
;* it under the terms of the GNU General Public License as published by
;* the Free Software Foundation; either version 2 of the License, or
;* (at your option) any later version.
;*
;* This program is distributed in the hope that it will be useful,
;* but WITHOUT ANY WARRANTY; without even the implied warranty of
;* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
;* GNU General Public License for more details.
;*
;* You should have received a copy of the GNU General Public License
;* along with this program; if not, write to the Free Software
;* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02111, USA.
;*
;* This program is also available under a commercial proprietary license.
;* For more information, contact us at licensing@x264.com.
;*****************************************************************************
%include "x86inc.asm"
SECTION_RODATA
error_message: db "failed to preserve register", 0
%if ARCH_X86_64
; just random numbers to reduce the chance of incidental match
ALIGN 16
x6: dq 0x1a1b2550a612b48c,0x79445c159ce79064
x7: dq 0x2eed899d5a28ddcd,0x86b2536fcd8cf636
x8: dq 0xb0856806085e7943,0x3f2bf84fc0fcca4e
x9: dq 0xacbd382dcf5b8de2,0xd229e1f5b281303f
x10: dq 0x71aeaff20b095fd9,0xab63e2e11fa38ed9
x11: dq 0x89b0c0765892729a,0x77d410d5c42c882d
x12: dq 0xc45ea11a955d8dd5,0x24b3c1d2a024048b
x13: dq 0x2e8ec680de14b47c,0xdd7b8919edd42786
x14: dq 0x135ce6888fa02cbf,0x11e53e2b2ac655ef
x15: dq 0x011ff554472a7a10,0x6de8f4c914c334d5
n7: dq 0x21f86d66c8ca00ce
n8: dq 0x75b6ba21077c48ad
n9: dq 0xed56bb2dcb3c7736
n10: dq 0x8bda43d3fd1a7e06
n11: dq 0xb64a9c9e5d318408
n12: dq 0xdf9a54b303f1d3a3
n13: dq 0x4a75479abd64e097
n14: dq 0x249214109d5d1c88
%endif
SECTION .text
cextern_naked puts
; max number of args used by any x264 asm function.
%define max_args 15
%if ARCH_X86_64
;-----------------------------------------------------------------------------
; void x264_checkasm_stack_clobber( uint64_t clobber, ... )
;-----------------------------------------------------------------------------
cglobal checkasm_stack_clobber, 1,2
; Clobber the stack with junk below the stack pointer
%define argsize (max_args+6)*8
SUB rsp, argsize
mov r1, argsize-8
.loop:
mov [rsp+r1], r0
sub r1, 8
jge .loop
ADD rsp, argsize
RET
%if WIN64
%assign free_regs 7
%else
%assign free_regs 9
%endif
;-----------------------------------------------------------------------------
; intptr_t x264_checkasm_call( intptr_t (*func)(), int *ok, ... )
;-----------------------------------------------------------------------------
INIT_XMM
cglobal checkasm_call, 2,15,16,-1*(((max_args+1)*8+STACK_ALIGNMENT-1) & ~(STACK_ALIGNMENT-1))
mov r6, r0
mov [rsp+max_args*8], r1
; All arguments have been pushed on the stack instead of registers in order to
; test for incorrect assumptions that 32-bit ints are zero-extended to 64-bit.
mov r0, r6mp
mov r1, r7mp
mov r2, r8mp
mov r3, r9mp
%if UNIX64
mov r4, r10mp
mov r5, r11mp
%assign i 6
%rep max_args-6
mov r9, [rstk+stack_offset+(i+1)*8]
mov [rsp+(i-6)*8], r9
%assign i i+1
%endrep
%else
%assign i 4
%rep max_args-4
mov r9, [rstk+stack_offset+(i+7)*8]
mov [rsp+i*8], r9
%assign i i+1
%endrep
%endif
%if WIN64
%assign i 6
%rep 16-6
mova m %+ i, [x %+ i]
%assign i i+1
%endrep
%endif
%assign i 14
%rep 15-free_regs
mov r %+ i, [n %+ i]
%assign i i-1
%endrep
call r6
%assign i 14
%rep 15-free_regs
xor r %+ i, [n %+ i]
or r14, r %+ i
%assign i i-1
%endrep
%if WIN64
%assign i 6
%rep 16-6
pxor m %+ i, [x %+ i]
por m6, m %+ i
%assign i i+1
%endrep
packsswb m6, m6
movq r5, m6
or r14, r5
%endif
jz .ok
mov r9, rax
mov r10, rdx
lea r0, [error_message]
call puts
mov r1, [rsp+max_args*8]
mov dword [r1], 0
mov rdx, r10
mov rax, r9
.ok:
RET
%else
; just random numbers to reduce the chance of incidental match
%define n3 dword 0x6549315c
%define n4 dword 0xe02f3e23
%define n5 dword 0xb78d0d1d
%define n6 dword 0x33627ba7
;-----------------------------------------------------------------------------
; intptr_t x264_checkasm_call( intptr_t (*func)(), int *ok, ... )
;-----------------------------------------------------------------------------
cglobal checkasm_call, 2,7,0,-1*(((max_args+1)*4+STACK_ALIGNMENT-1) & ~(STACK_ALIGNMENT-1))
mov [esp+max_args*4], r1
%assign i 0
%rep max_args
mov r1, [rstk+stack_offset+12+i*4]
mov [esp+i*4], r1
%assign i i+1
%endrep
mov r3, n3
mov r4, n4
mov r5, n5
mov r6, n6
call r0
xor r3, n3
xor r4, n4
xor r5, n5
xor r6, n6
or r3, r4
or r5, r6
or r3, r5
jz .ok
mov r3, eax
mov r4, edx
lea r1, [error_message]
mov [esp], r1
call puts
mov r1, [esp+max_args*4]
mov dword [r1], 0
mov edx, r4
mov eax, r3
.ok:
REP_RET
%endif ; ARCH_X86_64
;-----------------------------------------------------------------------------
; int x264_stack_pagealign( int (*func)(), int align )
;-----------------------------------------------------------------------------
cglobal stack_pagealign, 2,2
movsxdifnidn r1, r1d
push rbp
mov rbp, rsp
%if WIN64
sub rsp, 32 ; shadow space
%endif
and rsp, ~0xfff
sub rsp, r1
call r0
leave
RET
; Trigger a warmup of vector units
%macro WARMUP 0
cglobal checkasm_warmup, 0,0
xorps m0, m0
RET
%endmacro
INIT_YMM avx
WARMUP
INIT_ZMM avx512
WARMUP

178
tools/checkasm-aarch64.S Normal file
View File

@@ -0,0 +1,178 @@
/****************************************************************************
* checkasm-aarch64.S: assembly check tool
*****************************************************************************
* Copyright (C) 2015-2025 x264 project
*
* Authors: Martin Storsjo <martin@martin.st>
*
* This program is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation; either version 2 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program; if not, write to the Free Software
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02111, USA.
*
* This program is also available under a commercial proprietary license.
* For more information, contact us at licensing@x264.com.
*****************************************************************************/
#include "../common/aarch64/asm.S"
const register_init, align=4
.quad 0x21f86d66c8ca00ce
.quad 0x75b6ba21077c48ad
.quad 0xed56bb2dcb3c7736
.quad 0x8bda43d3fd1a7e06
.quad 0xb64a9c9e5d318408
.quad 0xdf9a54b303f1d3a3
.quad 0x4a75479abd64e097
.quad 0x249214109d5d1c88
.quad 0x1a1b2550a612b48c
.quad 0x79445c159ce79064
.quad 0x2eed899d5a28ddcd
.quad 0x86b2536fcd8cf636
.quad 0xb0856806085e7943
.quad 0x3f2bf84fc0fcca4e
.quad 0xacbd382dcf5b8de2
.quad 0xd229e1f5b281303f
.quad 0x71aeaff20b095fd9
.quad 0xab63e2e11fa38ed9
endconst
const error_message
.asciz "failed to preserve register"
endconst
.text
// max number of args used by any x264 asm function.
#define MAX_ARGS 15
#define CLOBBER_STACK ((8*MAX_ARGS + 15) & ~15)
function checkasm_stack_clobber, export=1
mov x3, sp
mov x2, #CLOBBER_STACK
1:
stp x0, x1, [sp, #-16]!
subs x2, x2, #16
b.gt 1b
mov sp, x3
ret
endfunc
#define ARG_STACK ((8*(MAX_ARGS - 8) + 15) & ~15)
function checkasm_call, export=1
stp x29, x30, [sp, #-16]!
mov x29, sp
stp x19, x20, [sp, #-16]!
stp x21, x22, [sp, #-16]!
stp x23, x24, [sp, #-16]!
stp x25, x26, [sp, #-16]!
stp x27, x28, [sp, #-16]!
stp d8, d9, [sp, #-16]!
stp d10, d11, [sp, #-16]!
stp d12, d13, [sp, #-16]!
stp d14, d15, [sp, #-16]!
movrel x9, register_init
ldp d8, d9, [x9], #16
ldp d10, d11, [x9], #16
ldp d12, d13, [x9], #16
ldp d14, d15, [x9], #16
ldp x19, x20, [x9], #16
ldp x21, x22, [x9], #16
ldp x23, x24, [x9], #16
ldp x25, x26, [x9], #16
ldp x27, x28, [x9], #16
str x1, [sp, #-16]!
sub sp, sp, #ARG_STACK
.equ pos, 0
.rept MAX_ARGS-8
// Skip the first 8 args, that are loaded into registers
ldr x9, [x29, #16 + 8*8 + pos]
str x9, [sp, #pos]
.equ pos, pos + 8
.endr
mov x12, x0
ldp x0, x1, [x29, #16]
ldp x2, x3, [x29, #32]
ldp x4, x5, [x29, #48]
ldp x6, x7, [x29, #64]
blr x12
add sp, sp, #ARG_STACK
ldr x2, [sp]
stp x0, x1, [sp]
movrel x9, register_init
movi v3.8h, #0
.macro check_reg_neon reg1, reg2
ldr q0, [x9], #16
uzp1 v1.2d, v\reg1\().2d, v\reg2\().2d
eor v0.16b, v0.16b, v1.16b
orr v3.16b, v3.16b, v0.16b
.endm
check_reg_neon 8, 9
check_reg_neon 10, 11
check_reg_neon 12, 13
check_reg_neon 14, 15
uqxtn v3.8b, v3.8h
umov x3, v3.d[0]
.macro check_reg reg1, reg2
ldp x0, x1, [x9], #16
eor x0, x0, \reg1
eor x1, x1, \reg2
orr x3, x3, x0
orr x3, x3, x1
.endm
check_reg x19, x20
check_reg x21, x22
check_reg x23, x24
check_reg x25, x26
check_reg x27, x28
cbz x3, 0f
mov w9, #0
str w9, [x2]
movrel x0, error_message
bl EXT(puts)
0:
ldp x0, x1, [sp], #16
ldp d14, d15, [sp], #16
ldp d12, d13, [sp], #16
ldp d10, d11, [sp], #16
ldp d8, d9, [sp], #16
ldp x27, x28, [sp], #16
ldp x25, x26, [sp], #16
ldp x23, x24, [sp], #16
ldp x21, x22, [sp], #16
ldp x19, x20, [sp], #16
ldp x29, x30, [sp], #16
ret
endfunc
#if HAVE_SVE
ENABLE_SVE
function checkasm_sve_length, export=1
cntb x0
lsl x0, x0, #3
ret
endfunc
DISABLE_SVE
#endif

142
tools/checkasm-arm.S Normal file
View File

@@ -0,0 +1,142 @@
/****************************************************************************
* checkasm-arm.S: assembly check tool
*****************************************************************************
* Copyright (C) 2015-2025 x264 project
*
* Authors: Martin Storsjo <martin@martin.st>
*
* This program is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation; either version 2 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program; if not, write to the Free Software
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02111, USA.
*
* This program is also available under a commercial proprietary license.
* For more information, contact us at licensing@x264.com.
*****************************************************************************/
#include "../common/arm/asm.S"
const register_init, align=4
.quad 0x21f86d66c8ca00ce
.quad 0x75b6ba21077c48ad
.quad 0xed56bb2dcb3c7736
.quad 0x8bda43d3fd1a7e06
.quad 0xb64a9c9e5d318408
.quad 0xdf9a54b303f1d3a3
.quad 0x4a75479abd64e097
.quad 0x249214109d5d1c88
endconst
const error_message
.asciz "failed to preserve register"
endconst
.text
@ max number of args used by any x264 asm function.
#define MAX_ARGS 15
#define ARG_STACK 4*(MAX_ARGS - 4)
@ align the used stack space to 8 to preserve the stack alignment
#define ARG_STACK_A (((ARG_STACK + pushed + 7) & ~7) - pushed)
.macro clobbercheck variant
.equ pushed, 4*10
function checkasm_call_\variant
push {r4-r11, lr}
.ifc \variant, neon
vpush {q4-q7}
.equ pushed, pushed + 16*4
.endif
movrel r12, register_init
.ifc \variant, neon
vldm r12, {q4-q7}
.endif
ldm r12, {r4-r11}
push {r1}
sub sp, sp, #ARG_STACK_A
.equ pos, 0
.rept MAX_ARGS-4
ldr r12, [sp, #ARG_STACK_A + pushed + 8 + pos]
str r12, [sp, #pos]
.equ pos, pos + 4
.endr
mov r12, r0
mov r0, r2
mov r1, r3
ldrd r2, r3, [sp, #ARG_STACK_A + pushed]
blx r12
add sp, sp, #ARG_STACK_A
pop {r2}
push {r0, r1}
movrel r12, register_init
.ifc \variant, neon
vldm r12, {q0-q3}
veor q0, q0, q4
veor q1, q1, q5
veor q2, q2, q6
veor q3, q3, q7
vorr q0, q0, q1
vorr q0, q0, q2
vorr q0, q0, q3
vorr d0, d0, d1
vrev64.32 d1, d0
vorr d0, d0, d1
vmov.32 r3, d0[0]
.else
mov r3, #0
.endif
.macro check_reg reg1, reg2=
ldrd r0, r1, [r12], #8
eor r0, r0, \reg1
orr r3, r3, r0
.ifnb \reg2
eor r1, r1, \reg2
orr r3, r3, r1
.endif
.endm
check_reg r4, r5
check_reg r6, r7
@ r9 is a volatile register in the ios ABI
#if SYS_MACOSX
check_reg r8
#else
check_reg r8, r9
#endif
check_reg r10, r11
.purgem check_reg
cmp r3, #0
beq 0f
mov r12, #0
str r12, [r2]
movrel r0, error_message
blx EXT(puts)
0:
pop {r0, r1}
.ifc \variant, neon
vpop {q4-q7}
.endif
pop {r4-r11, pc}
endfunc
.endm
clobbercheck neon
clobbercheck noneon

210
tools/checkasm-loongarch.S Normal file
View File

@@ -0,0 +1,210 @@
/****************************************************************************
* checkasm-loongarch.S: assembly check tool
*****************************************************************************
* Copyright (C) 2024-2025 x264 project
*
* Authors: Xiwei Gu <guxiwei-hf@loongson.cn>
*
* This program is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation; either version 2 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program; if not, write to the Free Software
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02111, USA.
*
* This program is also available under a commercial proprietary license.
* For more information, contact us at licensing@x264.com.
*****************************************************************************/
#include "../common/loongarch/loongson_asm.S"
const register_init, align=3
.quad 0x21f86d66c8ca00ce
.quad 0x75b6ba21077c48ad
.quad 0xed56bb2dcb3c7736
.quad 0x8bda43d3fd1a7e06
.quad 0xb64a9c9e5d318408
.quad 0xdf9a54b303f1d3a3
.quad 0x4a75479abd64e097
.quad 0x249214109d5d1c88
.quad 0x1a1b2550a612b48c
.quad 0x79445c159ce79064
.quad 0x2eed899d5a28ddcd
.quad 0x86b2536fcd8cf636
.quad 0xb0856806085e7943
.quad 0x3f2bf84fc0fcca4e
.quad 0xacbd382dcf5b8de2
.quad 0xd229e1f5b281303f
.quad 0x71aeaff20b095fd9
endconst
const error_message
.asciz "failed to preserve register"
endconst
.text
// max number of args used by any x264 asm function.
#define MAX_ARGS 15
#define CLOBBER_STACK ((8*MAX_ARGS + 15) & ~15)
// Fill dirty data at stack space
function x264_checkasm_stack_clobber
move t0, sp
addi.d t1, zero, CLOBBER_STACK
1:
st.d a0, sp, 0x00
st.d a1, sp, -0x08
addi.d sp, sp, -0x10
addi.d t1, t1, -0x10
blt zero,t1, 1b
move sp, t0
endfunc
#define ARG_STACK ((8*(MAX_ARGS - 8) + 15) & ~15)
function x264_checkasm_call
// Saved s0 - s8, fs0 - fs7
move t4, sp
addi.d sp, sp, -136
st.d s0, sp, 0
st.d s1, sp, 8
st.d s2, sp, 16
st.d s3, sp, 24
st.d s4, sp, 32
st.d s5, sp, 40
st.d s6, sp, 48
st.d s7, sp, 56
st.d s8, sp, 64
fst.d fs0, sp, 72
fst.d fs1, sp, 80
fst.d fs2, sp, 88
fst.d fs3, sp, 96
fst.d fs4, sp, 104
fst.d fs5, sp, 112
fst.d fs6, sp, 120
fst.d fs7, sp, 128
la.local t1, register_init
ld.d s0, t1, 0
ld.d s1, t1, 8
ld.d s2, t1, 16
ld.d s3, t1, 24
ld.d s4, t1, 32
ld.d s5, t1, 40
ld.d s6, t1, 48
ld.d s7, t1, 56
ld.d s8, t1, 64
fld.d fs0, t1, 72
fld.d fs1, t1, 80
fld.d fs2, t1, 88
fld.d fs3, t1, 96
fld.d fs4, t1, 104
fld.d fs5, t1, 112
fld.d fs6, t1, 120
fld.d fs7, t1, 128
addi.d sp, sp, -16
st.d a1, sp, 0 // ok
st.d ra, sp, 8 // Ret address
addi.d sp, sp, -ARG_STACK
addi.d t0, zero, 8*8
xor t1, t1, t1
.rept MAX_ARGS - 8
// Skip the first 8 args, that are loaded into registers
ldx.d t2, t4, t0
stx.d t2, sp, t1
addi.d t0, t0, 8
addi.d t1, t1, 8
.endr
move t3, a0 // Func
ld.d a0, t4, 0
ld.d a1, t4, 8
ld.d a2, t4, 16
ld.d a3, t4, 24
ld.d a4, t4, 32
ld.d a5, t4, 40
ld.d a6, t4, 48
ld.d a7, t4, 56
jirl ra, t3, 0
addi.d sp, sp, ARG_STACK
ld.d t2, sp, 0 // ok
ld.d ra, sp, 8 // Ret address
addi.d sp, sp, 16
la.local t1, register_init
xor t3, t3, t3
.macro check_reg_gr reg1
ld.d t0, t1, 0
xor t0, $s\reg1, t0
or t3, t3, t0
addi.d t1, t1, 8
.endm
check_reg_gr 0
check_reg_gr 1
check_reg_gr 2
check_reg_gr 3
check_reg_gr 4
check_reg_gr 5
check_reg_gr 6
check_reg_gr 7
check_reg_gr 8
.macro check_reg_fr reg1
ld.d t0, t1, 0
movfr2gr.d t4,$fs\reg1
xor t0, t0, t4
or t3, t3, t0
addi.d t1, t1, 8
.endm
check_reg_fr 0
check_reg_fr 1
check_reg_fr 2
check_reg_fr 3
check_reg_fr 4
check_reg_fr 5
check_reg_fr 6
check_reg_fr 7
beqz t3, 0f
st.d zero,t2, 0x00 // Set OK to 0
la.local a0, error_message
addi.d sp, sp, -8
st.d ra, sp, 0
bl puts
ld.d ra, sp, 0
addi.d sp, sp, 8
0:
ld.d s0, sp, 0
ld.d s1, sp, 8
ld.d s2, sp, 16
ld.d s3, sp, 24
ld.d s4, sp, 32
ld.d s5, sp, 40
ld.d s6, sp, 48
ld.d s7, sp, 56
ld.d s8, sp, 64
fld.d fs0, sp, 72
fld.d fs1, sp, 80
fld.d fs2, sp, 88
fld.d fs3, sp, 96
fld.d fs4, sp, 104
fld.d fs5, sp, 112
fld.d fs6, sp, 120
fld.d fs7, sp, 128
addi.d sp, sp, 136
endfunc

3098
tools/checkasm.c Normal file
View File

File diff suppressed because it is too large Load Diff

33
tools/cltostr.sh Executable file
View File

@@ -0,0 +1,33 @@
#!/bin/sh
# Convert standard input to a C char array, write to a file, then create an
# MD5 sum of that file and append said MD5 sum as char array to the file.
[ -n "$1" ] || exit 1
# Filter out whitespace, empty lines, and comments.
sanitize() {
sed 's/^[[:space:]]*//; /^$/d; /^\/\//d'
}
# Convert stdin to a \0-terminated char array.
dump() {
echo "static const char $1[] = {"
od -v -A n -t x1 | sed 's/[[:space:]]*\([[:alnum:]]\{2\}\)/0x\1, /g'
echo '0x00 };'
}
# Print MD5 hash w/o newline character to not embed the character in the array.
hash() {
# md5sum is not standard, so try different platform-specific alternatives.
{ md5sum "$1" || md5 -q "$1" || digest -a md5 "$1"; } 2>/dev/null |
cut -b -32 | tr -d '\n\r'
}
trap 'rm -f "$1.temp"' EXIT
sanitize | tee "$1.temp" |
dump 'x264_opencl_source' > "$1"
hash "$1.temp" |
dump 'x264_opencl_source_hash' >> "$1"

52
tools/countquant_x264.pl Executable file
View File

@@ -0,0 +1,52 @@
#!/bin/env perl
# countquant_x264.pl: displays statistics from x264 multipass logfiles
# by Loren Merritt, 2005-4-5
@size{I,P,B} =
@n{I,P,B} = (0)x3;
sub proc_file {
my $fh = shift;
while(<$fh>) {
/type:(.) q:(\d+\.\d+) tex:(\d+) mv:(\d+) misc:(\d+)/ or next;
$type = uc $1;
$n{$type} ++;
$q[int($2+.5)] ++;
$avgq += $2;
$avgq{$type} += $2;
my $bytes = ($3+$4+$5)/8;
$size{$type} += $bytes;
}
$size = $size{I} + $size{P} + $size{B};
$n = $n{I} + $n{P} + $n{B};
$n or die "unrecognized input\n";
}
if(@ARGV) {
foreach(@ARGV) {
open $fh, "<", $_ or die "can't open '$_': $!";
proc_file($fh);
}
} else {
proc_file(STDIN);
}
for(0..51) {
$q[$_] or next;
printf "q%2d: %6d %4.1f%%\n", $_, $q[$_], 100*$q[$_]/$n;
}
print "\n";
$digits = int(log($n+1)/log(10))+2;
printf "All: %${digits}d %s avgQP:%5.2f avgBytes:%5d\n",
$n, $n==$n{I}?" ":"", $avgq/$n, $size/$n;
foreach(qw(I P B S)) {
$n{$_} or next;
printf "%s: %${digits}d (%4.1f%%) avgQP:%5.2f avgBytes:%5d\n",
$_, $n{$_}, 100*$n{$_}/$n, $avgq{$_}/$n{$_}, $size{$_}/$n{$_};
}
print "\n";
printf "total size: $size B = %.2f KiB = %.2f MiB\n",
$size/2**10, $size/2**20;
print "bitrate: ", join("\n = ",
map sprintf("%.2f kbps @ %s fps", $_*$size*8/1000/$n, $_),
23.976, 25, 29.97), "\n";

12
tools/digress/__init__.py Normal file
View File

@@ -0,0 +1,12 @@
"""
Automated regression/unit testing suite.
"""
__version__ = '0.2'
def digress(fixture):
"""
Command-line helper for Digress.
"""
from digress.cli import Dispatcher
Dispatcher(fixture).dispatch()

149
tools/digress/cli.py Normal file
View File

@@ -0,0 +1,149 @@
"""
Digress's CLI interface.
"""
import inspect
import sys
from optparse import OptionParser
import textwrap
from types import MethodType
from digress import __version__ as version
def dispatchable(func):
"""
Mark a method as dispatchable.
"""
func.digress_dispatchable = True
return func
class Dispatcher(object):
"""
Dispatcher for CLI commands.
"""
def __init__(self, fixture):
self.fixture = fixture
fixture.dispatcher = self
def _monkey_print_help(self, optparse, *args, **kwargs):
# monkey patches OptionParser._print_help
OptionParser.print_help(optparse, *args, **kwargs)
print >>sys.stderr, "\nAvailable commands:"
maxlen = max([ len(command_name) for command_name in self.commands ])
descwidth = 80 - maxlen - 4
for command_name, command_meth in self.commands.iteritems():
print >>sys.stderr, " %s %s\n" % (
command_name.ljust(maxlen + 1),
("\n" + (maxlen + 4) * " ").join(
textwrap.wrap(" ".join(filter(
None,
command_meth.__doc__.strip().replace("\n", " ").split(" ")
)),
descwidth
)
)
)
def _enable_flush(self):
self.fixture.flush_before = True
def _populate_parser(self):
self.commands = self._get_commands()
self.optparse = OptionParser(
usage = "usage: %prog [options] command [args]",
description = "Digress CLI frontend for %s." % self.fixture.__class__.__name__,
version = "Digress %s" % version
)
self.optparse.print_help = MethodType(self._monkey_print_help, self.optparse, OptionParser)
self.optparse.add_option(
"-f",
"--flush",
action="callback",
callback=lambda option, opt, value, parser: self._enable_flush(),
help="flush existing data for a revision before testing"
)
self.optparse.add_option(
"-c",
"--cases",
metavar="FOO,BAR",
action="callback",
dest="cases",
type=str,
callback=lambda option, opt, value, parser: self._select_cases(*value.split(",")),
help="test cases to run, run with command list to see full list"
)
def _select_cases(self, *cases):
self.fixture.cases = filter(lambda case: case.__name__ in cases, self.fixture.cases)
def _get_commands(self):
commands = {}
for name, member in inspect.getmembers(self.fixture):
if hasattr(member, "digress_dispatchable"):
commands[name] = member
return commands
def _run_command(self, name, *args):
if name not in self.commands:
print >>sys.stderr, "error: %s is not a valid command\n" % name
self.optparse.print_help()
return
command = self.commands[name]
argspec = inspect.getargspec(command)
max_arg_len = len(argspec.args) - 1
min_arg_len = max_arg_len - ((argspec.defaults is not None) and len(argspec.defaults) or 0)
if len(args) < min_arg_len:
print >>sys.stderr, "error: %s takes at least %d arguments\n" % (
name,
min_arg_len
)
print >>sys.stderr, "%s\n" % command.__doc__
self.optparse.print_help()
return
if len(args) > max_arg_len:
print >>sys.stderr, "error: %s takes at most %d arguments\n" % (
name,
max_arg_len
)
print >>sys.stderr, "%s\n" % command.__doc__
self.optparse.print_help()
return
command(*args)
def pre_dispatch(self):
pass
def dispatch(self):
self._populate_parser()
self.optparse.parse_args()
self.pre_dispatch()
args = self.optparse.parse_args()[1] # arguments may require reparsing after pre_dispatch; see test_x264.py
if len(args) == 0:
print >>sys.stderr, "error: no command specified\n"
self.optparse.print_help()
return
command = args[0]
addenda = args[1:]
self._run_command(command, *addenda)

68
tools/digress/comparers.py Normal file
View File

@@ -0,0 +1,68 @@
"""
Digress comparers.
"""
from digress.errors import ComparisonError
import os
from itertools import imap, izip
def compare_direct(value_a, value_b):
if value_a != value_b:
raise ComparisonError("%s is not %s" % (value_a, value_b))
def compare_pass(value_a, value_b):
"""
Always true, as long as the test is passed.
"""
def compare_tolerance(tolerance):
def _compare_tolerance(value_a, value_b):
if abs(value_a - value_b) > tolerance:
raise ComparisonError("%s is not %s (tolerance: %s)" % (
value_a,
value_b,
tolerance
))
return _compare_tolerance
def compare_files(file_a, file_b):
size_a = os.path.getsize(file_a)
size_b = os.path.getsize(file_b)
print file_a, file_b
if size_a != size_b:
raise ComparisonError("%s is not the same size as %s" % (
file_a,
file_b
))
BUFFER_SIZE = 8196
offset = 0
with open(file_a) as f_a:
with open(file_b) as f_b:
for chunk_a, chunk_b in izip(
imap(
lambda i: f_a.read(BUFFER_SIZE),
xrange(size_a // BUFFER_SIZE + 1)
),
imap(
lambda i: f_b.read(BUFFER_SIZE),
xrange(size_b // BUFFER_SIZE + 1)
)
):
chunk_size = len(chunk_a)
if chunk_a != chunk_b:
for i in xrange(chunk_size):
if chunk_a[i] != chunk_b[i]:
raise ComparisonError("%s differs from %s at offset %d" % (
file_a,
file_b,
offset + i
))
offset += chunk_size

14
tools/digress/constants.py Normal file
View File

@@ -0,0 +1,14 @@
"""
All of Digress's constants.
"""
TEST_PASS = 0
TEST_FAIL = 1
TEST_DISABLED = 2
TEST_SKIPPED = 3
CASE_PASS = 0
CASE_FAIL = 1
FIXTURE_PASS = 0
FIXTURE_FAIL = 1

63
tools/digress/errors.py Normal file
View File

@@ -0,0 +1,63 @@
"""
Digress errors.
"""
class DigressError(Exception):
"""
Digress error base class.
"""
class NoSuchTestError(DigressError):
"""
Raised when no such test exists.
"""
class DisabledTestError(DigressError):
"""
Test is disabled.
"""
class SkippedTestError(DigressError):
"""
Test is marked as skipped.
"""
class DisabledCaseError(DigressError):
"""
Case is marked as disabled.
"""
class SkippedCaseError(DigressError):
"""
Case is marked as skipped.
"""
class FailedTestError(DigressError):
"""
Test failed.
"""
class ComparisonError(DigressError):
"""
Comparison failed.
"""
class IncomparableError(DigressError):
"""
Values cannot be compared.
"""
class AlreadyRunError(DigressError):
"""
Test/case has already been run.
"""
class SCMError(DigressError):
"""
Error occurred in SCM.
"""
def __init__(self, message):
self.message = message.replace("\n", " ")
def __str__(self):
return self.message

3
tools/digress/scm/__init__.py Normal file
View File

@@ -0,0 +1,3 @@
"""
Source control backends for Digress.
"""

41
tools/digress/scm/dummy.py Normal file
View File

@@ -0,0 +1,41 @@
"""
Dummy SCM backend for Digress.
"""
from random import random
def checkout(revision):
"""
Checkout a revision.
"""
pass
def current_rev():
"""
Get the current revision
"""
return str(random())
def revisions(rev_a, rev_b):
"""
Get a list of revisions from one to another.
"""
pass
def stash():
"""
Stash the repository.
"""
pass
def unstash():
"""
Unstash the repository.
"""
pass
def bisect(command, revision):
"""
Perform a bisection.
"""
raise NotImplementedError("dummy SCM backend does not support bisection")

119
tools/digress/scm/git.py Normal file
View File

@@ -0,0 +1,119 @@
"""
Git SCM backend for Digress.
"""
from subprocess import Popen, PIPE, STDOUT
import re
from digress.errors import SCMError
GIT_BRANCH_EXPR = re.compile("[*] (.*)")
def checkout(revision):
"""
Checkout a revision from git.
"""
proc = Popen([
"git",
"checkout",
"-f",
revision
], stdout=PIPE, stderr=STDOUT)
output = proc.communicate()[0].strip()
if proc.returncode != 0:
raise SCMError("checkout error: %s" % output)
def rev_parse(ref):
proc = Popen([
"git",
"rev-parse",
ref
], stdout=PIPE, stderr=STDOUT)
output = proc.communicate()[0].strip()
if proc.returncode != 0:
raise SCMError("rev-parse error: %s" % output)
return output
def current_rev():
"""
Get the current revision.
"""
return rev_parse("HEAD")
def current_branch():
"""
Get the current branch.
"""
proc = Popen([
"git",
"branch",
"--no-color"
], stdout=PIPE, stderr=STDOUT)
output = proc.communicate()[0].strip()
if proc.returncode != 0:
raise SCMError("branch error: %s" % output)
branch_name = GIT_BRANCH_EXPR.findall(output)[0]
return branch_name != "(no branch)" and branch_name or None
def revisions(rev_a, rev_b):
"""
Get a list of revisions from one to another.
"""
proc = Popen([
"git",
"log",
"--format=%H", ("%s...%s" % (rev_a, rev_b))
], stdout=PIPE, stderr=STDOUT)
output = proc.communicate()[0].strip()
if proc.returncode != 0:
raise SCMError("log error: %s" % output)
return output.split("\n")
def stash():
"""
Stash the repository.
"""
proc = Popen([
"git",
"stash",
"save",
"--keep-index"
], stdout=PIPE, stderr=STDOUT)
output = proc.communicate()[0].strip()
if proc.returncode != 0:
raise SCMError("stash error: %s" % output)
def unstash():
"""
Unstash the repository.
"""
proc = Popen(["git", "stash", "pop"], stdout=PIPE, stderr=STDOUT)
proc.communicate()
def bisect(*args):
"""
Perform a bisection.
"""
proc = Popen((["git", "bisect"] + list(args)), stdout=PIPE, stderr=STDOUT)
output = proc.communicate()[0]
if proc.returncode != 0:
raise SCMError("bisect error: %s" % output)
return output
def dirty():
"""
Check if the working tree is dirty.
"""
proc = Popen(["git", "status"], stdout=PIPE, stderr=STDOUT)
output = proc.communicate()[0].strip()
if proc.returncode != 0:
raise SCMError("status error: %s" % output)
if "modified:" in output:
return True
else:
return False

597
tools/digress/testing.py Normal file
View File

@@ -0,0 +1,597 @@
"""
Digress testing core.
"""
from digress.errors import SkippedTestError, DisabledTestError, NoSuchTestError, \
FailedTestError, AlreadyRunError, SCMError, \
ComparisonError
from digress.constants import *
from digress.cli import dispatchable
import inspect
import operator
import os
import json
import textwrap
from shutil import rmtree
from time import time
from functools import wraps
from itertools import izip_longest
from hashlib import sha1
class depends(object):
"""
Dependency decorator for a test.
"""
def __init__(self, *test_names):
self.test_names = test_names
def __call__(self, func):
func.digress_depends = self.test_names
return func
class _skipped(object):
"""
Internal skipped decorator.
"""
def __init__(self, reason=""):
self._reason = reason
def __call__(self, func):
@wraps(func)
def _closure(*args):
raise SkippedTestError(self._reason)
return _closure
class disabled(object):
"""
Disable a test, with reason.
"""
def __init__(self, reason=""):
self._reason = reason
def __call__(self, func):
@wraps(func)
def _closure(*args):
raise DisabledTestError(self._reason)
return _closure
class comparer(object):
"""
Set the comparer for a test.
"""
def __init__(self, comparer_):
self._comparer = comparer_
def __call__(self, func):
func.digress_comparer = self._comparer
return func
class Fixture(object):
cases = []
scm = None
flush_before = False
def _skip_case(self, case, depend):
for name, meth in inspect.getmembers(case):
if name[:5] == "test_":
setattr(
case,
name,
_skipped("failed dependency: case %s" % depend)(meth)
)
def _run_case(self, case, results):
if case.__name__ in results:
raise AlreadyRunError
for depend in case.depends:
if depend.__name__ in results and results[depend.__name__]["status"] != CASE_PASS:
self._skip_case(case, depend.__name__)
try:
result = self._run_case(depend, results)
except AlreadyRunError:
continue
if result["status"] != CASE_PASS:
self._skip_case(case, depend.__name__)
result = case().run()
results[case.__name__] = result
return result
@dispatchable
def flush(self, revision=None):
"""
Flush any cached results. Takes a revision for an optional argument.
"""
if not revision:
print "Flushing all cached results...",
try:
rmtree(".digress_%s" % self.__class__.__name__)
except Exception, e:
print "failed: %s" % e
else:
print "done."
else:
try:
rev = self.scm.rev_parse(revision)
except SCMError, e:
print e
else:
print "Flushing cached results for %s..." % rev,
try:
rmtree(os.path.join(".digress_%s" % self.__class__.__name__, rev))
except Exception, e:
print "failed: %s" % e
else:
print "done."
@dispatchable
def run(self, revision=None):
"""
Run the fixture for a specified revision.
Takes a revision for an argument.
"""
oldrev = None
oldbranch = None
dirty = False
try:
dirty = self.scm.dirty()
# if the tree is clean, then we don't need to make an exception
if not dirty and revision is None: revision = "HEAD"
if revision:
oldrev = self.scm.current_rev()
oldbranch = self.scm.current_branch()
if dirty:
self.scm.stash()
self.scm.checkout(revision)
rev = self.scm.current_rev()
self.datastore = os.path.join(".digress_%s" % self.__class__.__name__, rev)
if os.path.isdir(self.datastore):
if self.flush_before:
self.flush(rev)
else:
os.makedirs(self.datastore)
else:
rev = "(dirty working tree)"
self.datastore = None
print "Running fixture %s on revision %s...\n" % (self.__class__.__name__, rev)
results = {}
for case in self.cases:
try:
self._run_case(case, results)
except AlreadyRunError:
continue
total_time = reduce(operator.add, filter(
None,
[
result["time"] for result in results.values()
]
), 0)
overall_status = (
CASE_FAIL in [ result["status"] for result in results.values() ]
) and FIXTURE_FAIL or FIXTURE_PASS
print "Fixture %s in %.4f.\n" % (
(overall_status == FIXTURE_PASS) and "passed" or "failed",
total_time
)
return { "cases" : results, "time" : total_time, "status" : overall_status, "revision" : rev }
finally:
if oldrev:
self.scm.checkout(oldrev)
if oldbranch:
self.scm.checkout(oldbranch)
if dirty:
self.scm.unstash()
@dispatchable
def bisect(self, good_rev, bad_rev=None):
"""
Perform a bisection between two revisions.
First argument is the good revision, second is the bad revision, which
defaults to the current revision.
"""
if not bad_rev: bad_rev = self.scm.current_rev()
dirty = False
# get a set of results for the good revision
good_result = self.run(good_rev)
good_rev = good_result["revision"]
try:
dirty = self.scm.dirty()
if dirty:
self.scm.stash()
self.scm.bisect("start")
self.scm.bisect("bad", bad_rev)
self.scm.bisect("good", good_rev)
bisecting = True
isbad = False
while bisecting:
results = self.run(self.scm.current_rev())
revision = results["revision"]
# perform comparisons
# FIXME: this just uses a lot of self.compare
for case_name, case_result in good_result["cases"].iteritems():
case = filter(lambda case: case.__name__ == case_name, self.cases)[0]
for test_name, test_result in case_result["tests"].iteritems():
test = filter(
lambda pair: pair[0] == "test_%s" % test_name,
inspect.getmembers(case)
)[0][1]
other_result = results["cases"][case_name]["tests"][test_name]
if other_result["status"] == TEST_FAIL and case_result["status"] != TEST_FAIL:
print "Revision %s failed %s.%s." % (revision, case_name, test_name)
isbad = True
break
elif hasattr(test, "digress_comparer"):
try:
test.digress_comparer(test_result["value"], other_result["value"])
except ComparisonError, e:
print "%s differs: %s" % (test_name, e)
isbad = True
break
if isbad:
output = self.scm.bisect("bad", revision)
print "Marking revision %s as bad." % revision
else:
output = self.scm.bisect("good", revision)
print "Marking revision %s as good." % revision
if output.split("\n")[0].endswith("is the first bad commit"):
print "\nBisection complete.\n"
print output
bisecting = False
print ""
except SCMError, e:
print e
finally:
self.scm.bisect("reset")
if dirty:
self.scm.unstash()
@dispatchable
def multicompare(self, rev_a=None, rev_b=None, mode="waterfall"):
"""
Generate a comparison of tests.
Takes three optional arguments, from which revision, to which revision,
and the method of display (defaults to vertical "waterfall", also
accepts "river" for horizontal display)
"""
if not rev_a: rev_a = self.scm.current_rev()
if not rev_b: rev_b = self.scm.current_rev()
revisions = self.scm.revisions(rev_a, rev_b)
results = []
for revision in revisions:
results.append(self.run(revision))
test_names = reduce(operator.add, [
[
(case_name, test_name)
for
test_name, test_result
in
case_result["tests"].iteritems()
]
for
case_name, case_result
in
results[0]["cases"].iteritems()
], [])
MAXLEN = 20
colfmt = "| %s "
table = []
if mode not in ("waterfall", "river"):
mode = "waterfall"
print "Unknown multicompare mode specified, defaulting to %s." % mode
if mode == "waterfall":
header = [ "Test" ]
for result in results:
header.append(result["revision"])
table.append(header)
for test_name in test_names:
row_data = [ ".".join(test_name) ]
for result in results:
test_result = result["cases"][test_name[0]]["tests"][test_name[1]]
if test_result["status"] != TEST_PASS:
value = "did not pass: %s" % (test_result["value"])
else:
value = "%s (%.4f)" % (test_result["value"], test_result["time"])
row_data.append(value)
table.append(row_data)
elif mode == "river":
header = [ "Revision" ]
for test_name in test_names:
header.append(".".join(test_name))
table.append(header)
for result in results:
row_data = [ result["revision"] ]
for case_name, case_result in result["cases"].iteritems():
for test_name, test_result in case_result["tests"].iteritems():
if test_result["status"] != TEST_PASS:
value = "did not pass: %s" % (test_result["value"])
else:
value = "%s (%.4f)" % (test_result["value"], test_result["time"])
row_data.append(value)
table.append(row_data)
breaker = "=" * (len(colfmt % "".center(MAXLEN)) * len(table[0]) + 1)
print breaker
for row in table:
for row_stuff in izip_longest(*[
textwrap.wrap(col, MAXLEN, break_on_hyphens=False) for col in row
], fillvalue=""):
row_output = ""
for col in row_stuff:
row_output += colfmt % col.ljust(MAXLEN)
row_output += "|"
print row_output
print breaker
@dispatchable
def compare(self, rev_a, rev_b=None):
"""
Compare two revisions directly.
Takes two arguments, second is optional and implies current revision.
"""
results_a = self.run(rev_a)
results_b = self.run(rev_b)
for case_name, case_result in results_a["cases"].iteritems():
case = filter(lambda case: case.__name__ == case_name, self.cases)[0]
header = "Comparison of case %s" % case_name
print header
print "=" * len(header)
for test_name, test_result in case_result["tests"].iteritems():
test = filter(
lambda pair: pair[0] == "test_%s" % test_name,
inspect.getmembers(case)
)[0][1]
other_result = results_b["cases"][case_name]["tests"][test_name]
if test_result["status"] != TEST_PASS or other_result["status"] != TEST_PASS:
print "%s cannot be compared as one of the revisions have not passed it." % test_name
elif hasattr(test, "digress_comparer"):
try:
test.digress_comparer(test_result["value"], other_result["value"])
except ComparisonError, e:
print "%s differs: %s" % (test_name, e)
else:
print "%s does not differ." % test_name
else:
print "%s has no comparer and therefore cannot be compared." % test_name
print ""
@dispatchable
def list(self):
"""
List all available test cases, excluding dependencies.
"""
print "\nAvailable Test Cases"
print "===================="
for case in self.cases:
print case.__name__
def register_case(self, case):
case.fixture = self
self.cases.append(case)
class Case(object):
depends = []
fixture = None
def _get_test_by_name(self, test_name):
if not hasattr(self, "test_%s" % test_name):
raise NoSuchTestError(test_name)
return getattr(self, "test_%s" % test_name)
def _run_test(self, test, results):
test_name = test.__name__[5:]
if test_name in results:
raise AlreadyRunError
if hasattr(test, "digress_depends"):
for depend in test.digress_depends:
if depend in results and results[depend]["status"] != TEST_PASS:
test = _skipped("failed dependency: %s" % depend)(test)
dependtest = self._get_test_by_name(depend)
try:
result = self._run_test(dependtest, results)
except AlreadyRunError:
continue
if result["status"] != TEST_PASS:
test = _skipped("failed dependency: %s" % depend)(test)
start_time = time()
run_time = None
print "Running test %s..." % test_name,
try:
if not self.datastore:
# XXX: this smells funny
raise IOError
with open(os.path.join(
self.datastore,
"%s.json" % sha1(test_name).hexdigest()
), "r") as f:
result = json.load(f)
value = str(result["value"])
if result["status"] == TEST_DISABLED:
status = "disabled"
elif result["status"] == TEST_SKIPPED:
status = "skipped"
elif result["status"] == TEST_FAIL:
status = "failed"
elif result["status"] == TEST_PASS:
status = "passed"
value = "%s (in %.4f)" % (
result["value"] or "(no result)",
result["time"]
)
else:
status = "???"
print "%s (cached): %s" % (status, value)
except IOError:
try:
value = test()
except DisabledTestError, e:
print "disabled: %s" % e
status = TEST_DISABLED
value = str(e)
except SkippedTestError, e:
print "skipped: %s" % e
status = TEST_SKIPPED
value = str(e)
except FailedTestError, e:
print "failed: %s" % e
status = TEST_FAIL
value = str(e)
except Exception, e:
print "failed with exception: %s" % e
status = TEST_FAIL
value = str(e)
else:
run_time = time() - start_time
print "passed: %s (in %.4f)" % (
value or "(no result)",
run_time
)
status = TEST_PASS
result = { "status" : status, "value" : value, "time" : run_time }
if self.datastore:
with open(os.path.join(
self.datastore,
"%s.json" % sha1(test_name).hexdigest()
), "w") as f:
json.dump(result, f)
results[test_name] = result
return result
def run(self):
print "Running case %s..." % self.__class__.__name__
if self.fixture.datastore:
self.datastore = os.path.join(
self.fixture.datastore,
sha1(self.__class__.__name__).hexdigest()
)
if not os.path.isdir(self.datastore):
os.makedirs(self.datastore)
else:
self.datastore = None
results = {}
for name, meth in inspect.getmembers(self):
if name[:5] == "test_":
try:
self._run_test(meth, results)
except AlreadyRunError:
continue
total_time = reduce(operator.add, filter(
None, [
result["time"] for result in results.values()
]
), 0)
overall_status = (
TEST_FAIL in [ result["status"] for result in results.values() ]
) and CASE_FAIL or CASE_PASS
print "Case %s in %.4f.\n" % (
(overall_status == FIXTURE_PASS) and "passed" or "failed",
total_time
)
return { "tests" : results, "time" : total_time, "status" : overall_status }

1254
tools/gas-preprocessor.pl Executable file
View File

File diff suppressed because it is too large Load Diff

61
tools/msvsdepend.sh Executable file
View File

@@ -0,0 +1,61 @@
#!/bin/sh
# Output a Makefile rule describing the dependencies of a given source file.
# Expected arguments are $(CC) $(CFLAGS) $(SRC) $(OBJ)
set -f
[ -n "$1" ] && [ -n "$3" ] && [ -n "$4" ] || exit 1
# Add flags to only perform syntax checking and output a list of included files
# For sources that aren't C, run preprocessing to NUL instead.
case "$3" in
*.c)
opts="-W0 -Zs"
;;
*)
opts="-P -FiNUL"
;;
esac
# Discard all output other than included files
# Convert '\' directory separators to '/'
# Remove system includes (hack: check for "/Program Files" string in path)
# Add the source file itself as a dependency
deps="$($1 $2 -nologo -showIncludes $opts "$3" 2>&1 |
grep '^Note: including file:' |
sed 's/^Note: including file:[[:space:]]*\(.*\)$/\1/; s/\\/\//g' |
sed '/\/[Pp]rogram [Ff]iles/d')
$3"
# Convert Windows paths to Unix paths if possible
if command -v cygpath >/dev/null 2>&1 ; then
IFS='
'
deps="$(cygpath -u -- $deps)"
elif grep -q 'Microsoft' /proc/sys/kernel/osrelease 2>/dev/null ; then
# Running under WSL. We don't have access to cygpath but since the Windows
# file system resides under "/mnt/<drive_letter>/" we can simply replace
# "C:" with "/mnt/c". This command uses a GNU extension to sed but that's
# available on WSL so we don't need to limit ourselves by what POSIX says.
deps="$(printf '%s' "$deps" | sed 's/^\([a-zA-Z]\):/\/mnt\/\L\1/')"
fi
# Escape characters as required to create valid Makefile file names
escape() {
sed 's/ /\\ /g; s/#/\\#/g; s/\$/\$\$/g'
}
# Remove prefixes that are equal to the working directory
# Sort and remove duplicate entries
# Escape and collapse the dependencies into one line
deps="$(printf '%s' "$deps" |
sed "s/^$(pwd | sed 's/\//\\\//g')\///; s/^\.\///" |
sort | uniq |
escape | tr -s '\n\r' ' ' | sed 's/^ *\(.*\) $/\1/')"
# Escape the target file name as well
target="$(printf '%s' "$4" | escape)"
printf '%s: %s\n' "$target" "$deps"

68
tools/q_matrix_jvt.cfg Normal file
View File

@@ -0,0 +1,68 @@
# This an example configuration file for initializing the quantization matrix.
# Altogether 6 matrices for 4x4 blocks and 2 matrix for 8x8 blocks.
# The values range from 1 to 255.
# If first value of matrix is equal to 0, default values ("JVT") will be used
# for that matrix.
# If a matrix is completely omitted, it will be filled with 16s.
#
# Note: JM expects CHROMAU and CHROMAV to be specified separately, whereas
# x264 forces them to use the same matrix. If U and V are specified to have
# different matrices, only the first is used.
####################################################################################
INTRA4X4_LUMA =
6,13,20,28,
13,20,28,32,
20,28,32,37,
28,32,37,42
INTRA4X4_CHROMAU =
6,13,20,28,
13,20,28,32,
20,28,32,37,
28,32,37,42
INTRA4X4_CHROMAV =
6,13,20,28,
13,20,28,32,
20,28,32,37,
28,32,37,42
INTER4X4_LUMA =
10,14,20,24,
14,20,24,27,
20,24,27,30,
24,27,30,34
INTER4X4_CHROMAU =
10,14,20,24,
14,20,24,27,
20,24,27,30,
24,27,30,34
INTER4X4_CHROMAV =
10,14,20,24,
14,20,24,27,
20,24,27,30,
24,27,30,34
INTRA8X8_LUMA =
6,10,13,16,18,23,25,27,
10,11,16,18,23,25,27,29,
13,16,18,23,25,27,29,31,
16,18,23,25,27,29,31,33,
18,23,25,27,29,31,33,36,
23,25,27,29,31,33,36,38,
25,27,29,31,33,36,38,40,
27,29,31,33,36,38,40,42
INTER8X8_LUMA =
9,13,15,17,19,21,22,24,
13,13,17,19,21,22,24,25,
15,17,19,21,22,24,25,27,
17,19,21,22,24,25,27,28,
19,21,22,24,25,27,28,30,
21,22,24,25,27,28,30,32,
22,24,25,27,28,30,32,33,
24,25,27,28,30,32,33,35

485
tools/test_x264.py Executable file
View File

@@ -0,0 +1,485 @@
#!/usr/bin/env python
import operator
from optparse import OptionGroup
import sys
from time import time
from digress.cli import Dispatcher as _Dispatcher
from digress.errors import ComparisonError, FailedTestError, DisabledTestError
from digress.testing import depends, comparer, Fixture, Case
from digress.comparers import compare_pass
from digress.scm import git as x264git
from subprocess import Popen, PIPE, STDOUT
import os
import re
import shlex
import inspect
from random import randrange, seed
from math import ceil
from itertools import imap, izip
os.chdir(os.path.join(os.path.dirname(__file__), ".."))
# options
OPTIONS = [
[ "--tune %s" % t for t in ("film", "zerolatency") ],
("", "--intra-refresh"),
("", "--no-cabac"),
("", "--interlaced"),
("", "--slice-max-size 1000"),
("", "--frame-packing 5"),
[ "--preset %s" % p for p in ("ultrafast",
"superfast",
"veryfast",
"faster",
"fast",
"medium",
"slow",
"slower",
"veryslow",
"placebo") ]
]
# end options
def compare_yuv_output(width, height):
def _compare_yuv_output(file_a, file_b):
size_a = os.path.getsize(file_a)
size_b = os.path.getsize(file_b)
if size_a != size_b:
raise ComparisonError("%s is not the same size as %s" % (
file_a,
file_b
))
BUFFER_SIZE = 8196
offset = 0
with open(file_a) as f_a:
with open(file_b) as f_b:
for chunk_a, chunk_b in izip(
imap(
lambda i: f_a.read(BUFFER_SIZE),
xrange(size_a // BUFFER_SIZE + 1)
),
imap(
lambda i: f_b.read(BUFFER_SIZE),
xrange(size_b // BUFFER_SIZE + 1)
)
):
chunk_size = len(chunk_a)
if chunk_a != chunk_b:
for i in xrange(chunk_size):
if chunk_a[i] != chunk_b[i]:
# calculate the macroblock, plane and frame from the offset
offs = offset + i
y_plane_area = width * height
u_plane_area = y_plane_area + y_plane_area * 0.25
v_plane_area = u_plane_area + y_plane_area * 0.25
pixel = offs % v_plane_area
frame = offs // v_plane_area
if pixel < y_plane_area:
plane = "Y"
pixel_x = pixel % width
pixel_y = pixel // width
macroblock = (ceil(pixel_x / 16.0), ceil(pixel_y / 16.0))
elif pixel < u_plane_area:
plane = "U"
pixel -= y_plane_area
pixel_x = pixel % width
pixel_y = pixel // width
macroblock = (ceil(pixel_x / 8.0), ceil(pixel_y / 8.0))
else:
plane = "V"
pixel -= u_plane_area
pixel_x = pixel % width
pixel_y = pixel // width
macroblock = (ceil(pixel_x / 8.0), ceil(pixel_y / 8.0))
macroblock = tuple([ int(x) for x in macroblock ])
raise ComparisonError("%s differs from %s at frame %d, " \
"macroblock %s on the %s plane (offset %d)" % (
file_a,
file_b,
frame,
macroblock,
plane,
offs)
)
offset += chunk_size
return _compare_yuv_output
def program_exists(program):
def is_exe(fpath):
return os.path.exists(fpath) and os.access(fpath, os.X_OK)
fpath, fname = os.path.split(program)
if fpath:
if is_exe(program):
return program
else:
for path in os.environ["PATH"].split(os.pathsep):
exe_file = os.path.join(path, program)
if is_exe(exe_file):
return exe_file
return None
class x264(Fixture):
scm = x264git
class Compile(Case):
@comparer(compare_pass)
def test_configure(self):
Popen([
"make",
"distclean"
], stdout=PIPE, stderr=STDOUT).communicate()
configure_proc = Popen([
"./configure"
] + self.fixture.dispatcher.configure, stdout=PIPE, stderr=STDOUT)
output = configure_proc.communicate()[0]
if configure_proc.returncode != 0:
raise FailedTestError("configure failed: %s" % output.replace("\n", " "))
@depends("configure")
@comparer(compare_pass)
def test_make(self):
make_proc = Popen([
"make",
"-j5"
], stdout=PIPE, stderr=STDOUT)
output = make_proc.communicate()[0]
if make_proc.returncode != 0:
raise FailedTestError("make failed: %s" % output.replace("\n", " "))
_dimension_pattern = re.compile(r"\w+ [[]info[]]: (\d+)x(\d+)[pi] \d+:\d+ @ \d+/\d+ fps [(][vc]fr[)]")
def _YUVOutputComparisonFactory():
class YUVOutputComparison(Case):
_dimension_pattern = _dimension_pattern
depends = [ Compile ]
options = []
def __init__(self):
for name, meth in inspect.getmembers(self):
if name[:5] == "test_" and name[5:] not in self.fixture.dispatcher.yuv_tests:
delattr(self.__class__, name)
def _run_x264(self):
x264_proc = Popen([
"./x264",
"-o",
"%s.264" % self.fixture.dispatcher.video,
"--dump-yuv",
"x264-output.yuv"
] + self.options + [
self.fixture.dispatcher.video
], stdout=PIPE, stderr=STDOUT)
output = x264_proc.communicate()[0]
if x264_proc.returncode != 0:
raise FailedTestError("x264 did not complete properly: %s" % output.replace("\n", " "))
matches = _dimension_pattern.match(output)
return (int(matches.group(1)), int(matches.group(2)))
@comparer(compare_pass)
def test_jm(self):
if not program_exists("ldecod"): raise DisabledTestError("jm unavailable")
try:
runres = self._run_x264()
jm_proc = Popen([
"ldecod",
"-i",
"%s.264" % self.fixture.dispatcher.video,
"-o",
"jm-output.yuv"
], stdout=PIPE, stderr=STDOUT)
output = jm_proc.communicate()[0]
if jm_proc.returncode != 0:
raise FailedTestError("jm did not complete properly: %s" % output.replace("\n", " "))
try:
compare_yuv_output(*runres)("x264-output.yuv", "jm-output.yuv")
except ComparisonError, e:
raise FailedTestError(e)
finally:
try: os.remove("x264-output.yuv")
except: pass
try: os.remove("%s.264" % self.fixture.dispatcher.video)
except: pass
try: os.remove("jm-output.yuv")
except: pass
try: os.remove("log.dec")
except: pass
try: os.remove("dataDec.txt")
except: pass
@comparer(compare_pass)
def test_ffmpeg(self):
if not program_exists("ffmpeg"): raise DisabledTestError("ffmpeg unavailable")
try:
runres = self._run_x264()
ffmpeg_proc = Popen([
"ffmpeg",
"-vsync 0",
"-i",
"%s.264" % self.fixture.dispatcher.video,
"ffmpeg-output.yuv"
], stdout=PIPE, stderr=STDOUT)
output = ffmpeg_proc.communicate()[0]
if ffmpeg_proc.returncode != 0:
raise FailedTestError("ffmpeg did not complete properly: %s" % output.replace("\n", " "))
try:
compare_yuv_output(*runres)("x264-output.yuv", "ffmpeg-output.yuv")
except ComparisonError, e:
raise FailedTestError(e)
finally:
try: os.remove("x264-output.yuv")
except: pass
try: os.remove("%s.264" % self.fixture.dispatcher.video)
except: pass
try: os.remove("ffmpeg-output.yuv")
except: pass
return YUVOutputComparison
class Regression(Case):
depends = [ Compile ]
_psnr_pattern = re.compile(r"x264 [[]info[]]: PSNR Mean Y:\d+[.]\d+ U:\d+[.]\d+ V:\d+[.]\d+ Avg:\d+[.]\d+ Global:(\d+[.]\d+) kb/s:\d+[.]\d+")
_ssim_pattern = re.compile(r"x264 [[]info[]]: SSIM Mean Y:(\d+[.]\d+) [(]\d+[.]\d+db[)]")
def __init__(self):
if self.fixture.dispatcher.x264:
self.__class__.__name__ += " %s" % " ".join(self.fixture.dispatcher.x264)
def test_psnr(self):
try:
x264_proc = Popen([
"./x264",
"-o",
"%s.264" % self.fixture.dispatcher.video,
"--psnr"
] + self.fixture.dispatcher.x264 + [
self.fixture.dispatcher.video
], stdout=PIPE, stderr=STDOUT)
output = x264_proc.communicate()[0]
if x264_proc.returncode != 0:
raise FailedTestError("x264 did not complete properly: %s" % output.replace("\n", " "))
for line in output.split("\n"):
if line.startswith("x264 [info]: PSNR Mean"):
return float(self._psnr_pattern.match(line).group(1))
raise FailedTestError("no PSNR output caught from x264")
finally:
try: os.remove("%s.264" % self.fixture.dispatcher.video)
except: pass
def test_ssim(self):
try:
x264_proc = Popen([
"./x264",
"-o",
"%s.264" % self.fixture.dispatcher.video,
"--ssim"
] + self.fixture.dispatcher.x264 + [
self.fixture.dispatcher.video
], stdout=PIPE, stderr=STDOUT)
output = x264_proc.communicate()[0]
if x264_proc.returncode != 0:
raise FailedTestError("x264 did not complete properly: %s" % output.replace("\n", " "))
for line in output.split("\n"):
if line.startswith("x264 [info]: SSIM Mean"):
return float(self._ssim_pattern.match(line).group(1))
raise FailedTestError("no PSNR output caught from x264")
finally:
try: os.remove("%s.264" % self.fixture.dispatcher.video)
except: pass
def _generate_random_commandline():
commandline = []
for suboptions in OPTIONS:
commandline.append(suboptions[randrange(0, len(suboptions))])
return filter(None, reduce(operator.add, [ shlex.split(opt) for opt in commandline ]))
_generated = []
fixture = x264()
fixture.register_case(Compile)
fixture.register_case(Regression)
class Dispatcher(_Dispatcher):
video = "akiyo_qcif.y4m"
products = 50
configure = []
x264 = []
yuv_tests = [ "jm" ]
def _populate_parser(self):
super(Dispatcher, self)._populate_parser()
# don't do a whole lot with this
tcase = _YUVOutputComparisonFactory()
yuv_tests = [ name[5:] for name, meth in filter(lambda pair: pair[0][:5] == "test_", inspect.getmembers(tcase)) ]
group = OptionGroup(self.optparse, "x264 testing-specific options")
group.add_option(
"-v",
"--video",
metavar="FILENAME",
action="callback",
dest="video",
type=str,
callback=lambda option, opt, value, parser: setattr(self, "video", value),
help="yuv video to perform testing on (default: %s)" % self.video
)
group.add_option(
"-s",
"--seed",
metavar="SEED",
action="callback",
dest="seed",
type=int,
callback=lambda option, opt, value, parser: setattr(self, "seed", value),
help="seed for the random number generator (default: unix timestamp)"
)
group.add_option(
"-p",
"--product-tests",
metavar="NUM",
action="callback",
dest="video",
type=int,
callback=lambda option, opt, value, parser: setattr(self, "products", value),
help="number of cartesian products to generate for yuv comparison testing (default: %d)" % self.products
)
group.add_option(
"--configure-with",
metavar="FLAGS",
action="callback",
dest="configure",
type=str,
callback=lambda option, opt, value, parser: setattr(self, "configure", shlex.split(value)),
help="options to run ./configure with"
)
group.add_option(
"--yuv-tests",
action="callback",
dest="yuv_tests",
type=str,
callback=lambda option, opt, value, parser: setattr(self, "yuv_tests", [
val.strip() for val in value.split(",")
]),
help="select tests to run with yuv comparisons (default: %s, available: %s)" % (
", ".join(self.yuv_tests),
", ".join(yuv_tests)
)
)
group.add_option(
"--x264-with",
metavar="FLAGS",
action="callback",
dest="x264",
type=str,
callback=lambda option, opt, value, parser: setattr(self, "x264", shlex.split(value)),
help="additional options to run ./x264 with"
)
self.optparse.add_option_group(group)
def pre_dispatch(self):
if not hasattr(self, "seed"):
self.seed = int(time())
print "Using seed: %d" % self.seed
seed(self.seed)
for i in xrange(self.products):
YUVOutputComparison = _YUVOutputComparisonFactory()
commandline = _generate_random_commandline()
counter = 0
while commandline in _generated:
counter += 1
commandline = _generate_random_commandline()
if counter > 100:
print >>sys.stderr, "Maximum command-line regeneration exceeded. " \
"Try a different seed or specify fewer products to generate."
sys.exit(1)
commandline += self.x264
_generated.append(commandline)
YUVOutputComparison.options = commandline
YUVOutputComparison.__name__ = ("%s %s" % (YUVOutputComparison.__name__, " ".join(commandline)))
fixture.register_case(YUVOutputComparison)
Dispatcher(fixture).dispatch()