Skip to content

Commit

Permalink
Remove usage of LoggingExtras and verbosity logging macros (#529)
Browse files Browse the repository at this point in the history
  • Loading branch information
@quinnj
quinnj authored Oct 28, 2024
1 parent a31fcf9 commit aa5ad74
Show file tree
Hide file tree
Showing 14 changed files with 96 additions and 99 deletions.
2 changes: 0 additions & 2 deletions Project.toml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,6 @@ ConcurrentUtilities = "f0e56b4a-5159-44fe-b623-3e5288b988bb"
DataAPI = "9a962f9c-6df0-11e9-0e5d-c546b8b5ee8a"
Dates = "ade2ca70-3891-5945-98fb-dc099432e06a"
EnumX = "4e289a0a-7415-4d19-859d-a7e5c4648b56"
LoggingExtras = "e6f89c97-d47a-5376-807f-9c37f3926c36"
Mmap = "a63ad114-7e13-5084-954f-fe012c677804"
PooledArrays = "2dfb63ee-cc39-5dd5-95bd-886bf059d720"
SentinelArrays = "91c51154-3ec4-41a3-a24f-3f23e20d615c"
Expand All @@ -45,7 +44,6 @@ CodecZstd = "0.7, 0.8"
ConcurrentUtilities = "2"
DataAPI = "1"
EnumX = "1"
LoggingExtras = "0.4, 1"
PooledArrays = "0.5, 1.0"
SentinelArrays = "1"
Tables = "1.1"
Expand Down
1 change: 0 additions & 1 deletion src/Arrow.jl
View file
Original file line number Diff line number Diff line change
Expand Up @@ -43,7 +43,6 @@ module Arrow

using Base.Iterators
using Mmap
using LoggingExtras
import Dates
using DataAPI,
Tables,
Expand Down
2 changes: 1 addition & 1 deletion src/append.jl
View file
Original file line number Diff line number Diff line change
Expand Up @@ -182,7 +182,7 @@ function append(
(errorref[][1], errorref[][2])
error("fatal error writing arrow data")
end
@debugv 1 "processing table partition i = $i"
@debug "processing table partition i = $i"
tbl_cols = Tables.columns(tbl)
tbl_schema = Tables.schema(tbl_cols)

Expand Down
12 changes: 6 additions & 6 deletions src/arraytypes/arraytypes.jl
View file
Original file line number Diff line number Diff line change
Expand Up @@ -42,8 +42,8 @@ function toarrowvector(
compression::Union{Nothing,Symbol,LZ4FrameCompressor,ZstdCompressor}=nothing,
kw...,
)
@debugv 2 "converting top-level column to arrow format: col = $(typeof(x)), compression = $compression, kw = $(values(kw))"
@debugv 3 x
@debug "converting top-level column to arrow format: col = $(typeof(x)), compression = $compression, kw = $(values(kw))"
@debug x
A = arrowvector(x, i, 0, 0, de, ded, meta; compression=compression, kw...)
if compression isa LZ4FrameCompressor
A = compress(Meta.CompressionType.LZ4_FRAME, compression, A)
Expand All @@ -60,8 +60,8 @@ function toarrowvector(
compress(Meta.CompressionType.ZSTD, comp[], A)
end
end
@debugv 2 "converted top-level column to arrow format: $(typeof(A))"
@debugv 3 A
@debug "converted top-level column to arrow format: $(typeof(A))"
@debug A
return A
end

Expand Down Expand Up @@ -173,7 +173,7 @@ function makenodesbuffers!(
alignment,
)
push!(fieldnodes, FieldNode(length(col), length(col)))
@debugv 1 "made field node: nodeidx = $(length(fieldnodes)), col = $(typeof(col)), len = $(fieldnodes[end].length), nc = $(fieldnodes[end].null_count)"
@debug "made field node: nodeidx = $(length(fieldnodes)), col = $(typeof(col)), len = $(fieldnodes[end].length), nc = $(fieldnodes[end].null_count)"
return bufferoffset
end

Expand Down Expand Up @@ -256,7 +256,7 @@ end

function writebitmap(io, col::ArrowVector, alignment)
v = col.validity
@debugv 1 "writing validity bitmap: nc = $(v.nc), n = $(cld(v.ℓ, 8))"
@debug "writing validity bitmap: nc = $(v.nc), n = $(cld(v.ℓ, 8))"
v.nc == 0 && return 0
n = Base.write(io, view(v.bytes, (v.pos):(v.pos + cld(v.ℓ, 8) - 1)))
return n + writezeros(io, paddinglength(n, alignment))
Expand Down
10 changes: 5 additions & 5 deletions src/arraytypes/bool.jl
View file
Original file line number Diff line number Diff line change
Expand Up @@ -95,22 +95,22 @@ function makenodesbuffers!(
len = length(col)
nc = nullcount(col)
push!(fieldnodes, FieldNode(len, nc))
@debugv 1 "made field node: nodeidx = $(length(fieldnodes)), col = $(typeof(col)), len = $(fieldnodes[end].length), nc = $(fieldnodes[end].null_count)"
@debug "made field node: nodeidx = $(length(fieldnodes)), col = $(typeof(col)), len = $(fieldnodes[end].length), nc = $(fieldnodes[end].null_count)"
# validity bitmap
blen = nc == 0 ? 0 : bitpackedbytes(len, alignment)
push!(fieldbuffers, Buffer(bufferoffset, blen))
@debugv 1 "made field buffer: bufferidx = $(length(fieldbuffers)), offset = $(fieldbuffers[end].offset), len = $(fieldbuffers[end].length), padded = $(padding(fieldbuffers[end].length, alignment))"
@debug "made field buffer: bufferidx = $(length(fieldbuffers)), offset = $(fieldbuffers[end].offset), len = $(fieldbuffers[end].length), padded = $(padding(fieldbuffers[end].length, alignment))"
# adjust buffer offset, make primitive array buffer
bufferoffset += blen
blen = bitpackedbytes(len, alignment)
push!(fieldbuffers, Buffer(bufferoffset, blen))
@debugv 1 "made field buffer: bufferidx = $(length(fieldbuffers)), offset = $(fieldbuffers[end].offset), len = $(fieldbuffers[end].length), padded = $(padding(fieldbuffers[end].length, alignment))"
@debug "made field buffer: bufferidx = $(length(fieldbuffers)), offset = $(fieldbuffers[end].offset), len = $(fieldbuffers[end].length), padded = $(padding(fieldbuffers[end].length, alignment))"
return bufferoffset + blen
end

function writebuffer(io, col::BoolVector, alignment)
@debugv 1 "writebuffer: col = $(typeof(col))"
@debugv 2 col
@debug "writebuffer: col = $(typeof(col))"
@debug col
writebitmap(io, col, alignment)
n = Base.write(io, view(col.arrow, (col.pos):(col.pos + cld(col.ℓ, 8) - 1)))
return n + writezeros(io, paddinglength(n, alignment))
Expand Down
12 changes: 6 additions & 6 deletions src/arraytypes/compressed.jl
View file
Original file line number Diff line number Diff line change
Expand Up @@ -60,11 +60,11 @@ function makenodesbuffers!(
alignment,
)
push!(fieldnodes, FieldNode(col.len, col.nullcount))
@debugv 1 "made field node: nodeidx = $(length(fieldnodes)), col = $(typeof(col)), len = $(fieldnodes[end].length), nc = $(fieldnodes[end].null_count)"
@debug "made field node: nodeidx = $(length(fieldnodes)), col = $(typeof(col)), len = $(fieldnodes[end].length), nc = $(fieldnodes[end].null_count)"
for buffer in col.buffers
blen = length(buffer.data) == 0 ? 0 : 8 + length(buffer.data)
push!(fieldbuffers, Buffer(bufferoffset, blen))
@debugv 1 "made field buffer: bufferidx = $(length(fieldbuffers)), offset = $(fieldbuffers[end].offset), len = $(fieldbuffers[end].length), padded = $(padding(fieldbuffers[end].length, alignment))"
@debug "made field buffer: bufferidx = $(length(fieldbuffers)), offset = $(fieldbuffers[end].offset), len = $(fieldbuffers[end].length), padded = $(padding(fieldbuffers[end].length, alignment))"
bufferoffset += padding(blen, alignment)
end
for child in col.children
Expand All @@ -77,16 +77,16 @@ end
function writearray(io, b::CompressedBuffer)
if length(b.data) > 0
n = Base.write(io, b.uncompressedlength)
@debugv 1 "writing compressed buffer: uncompressedlength = $(b.uncompressedlength), n = $(length(b.data))"
@debugv 2 b.data
@debug "writing compressed buffer: uncompressedlength = $(b.uncompressedlength), n = $(length(b.data))"
@debug b.data
return n + Base.write(io, b.data)
end
return 0
end

function writebuffer(io, col::Compressed, alignment)
@debugv 1 "writebuffer: col = $(typeof(col))"
@debugv 2 col
@debug "writebuffer: col = $(typeof(col))"
@debug col
for buffer in col.buffers
n = writearray(io, buffer)
writezeros(io, paddinglength(n, alignment))
Expand Down
12 changes: 6 additions & 6 deletions src/arraytypes/dictencoding.jl
View file
Original file line number Diff line number Diff line change
Expand Up @@ -402,16 +402,16 @@ function makenodesbuffers!(
len = length(col)
nc = nullcount(col)
push!(fieldnodes, FieldNode(len, nc))
@debugv 1 "made field node: nodeidx = $(length(fieldnodes)), col = $(typeof(col)), len = $(fieldnodes[end].length), nc = $(fieldnodes[end].null_count)"
@debug "made field node: nodeidx = $(length(fieldnodes)), col = $(typeof(col)), len = $(fieldnodes[end].length), nc = $(fieldnodes[end].null_count)"
# validity bitmap
blen = nc == 0 ? 0 : bitpackedbytes(len, alignment)
push!(fieldbuffers, Buffer(bufferoffset, blen))
@debugv 1 "made field buffer: bufferidx = $(length(fieldbuffers)), offset = $(fieldbuffers[end].offset), len = $(fieldbuffers[end].length), padded = $(padding(fieldbuffers[end].length, alignment))"
@debug "made field buffer: bufferidx = $(length(fieldbuffers)), offset = $(fieldbuffers[end].offset), len = $(fieldbuffers[end].length), padded = $(padding(fieldbuffers[end].length, alignment))"
bufferoffset += blen
# indices
blen = sizeof(S) * len
push!(fieldbuffers, Buffer(bufferoffset, blen))
@debugv 1 "made field buffer: bufferidx = $(length(fieldbuffers)), offset = $(fieldbuffers[end].offset), len = $(fieldbuffers[end].length), padded = $(padding(fieldbuffers[end].length, alignment))"
@debug "made field buffer: bufferidx = $(length(fieldbuffers)), offset = $(fieldbuffers[end].offset), len = $(fieldbuffers[end].length), padded = $(padding(fieldbuffers[end].length, alignment))"
bufferoffset += padding(blen, alignment)
return bufferoffset
end
Expand All @@ -421,12 +421,12 @@ DataAPI.refarray(x::DictEncoded{T,S}) where {T,S} = x.indices .+ one(S)
DataAPI.refpool(x::DictEncoded) = copy(x.encoding.data)

function writebuffer(io, col::DictEncoded, alignment)
@debugv 1 "writebuffer: col = $(typeof(col))"
@debugv 2 col
@debug "writebuffer: col = $(typeof(col))"
@debug col
writebitmap(io, col, alignment)
# write indices
n = writearray(io, col.indices)
@debugv 1 "writing array: col = $(typeof(col.indices)), n = $n, padded = $(padding(n, alignment))"
@debug "writing array: col = $(typeof(col.indices)), n = $n, padded = $(padding(n, alignment))"
writezeros(io, paddinglength(n, alignment))
return
end
12 changes: 6 additions & 6 deletions src/arraytypes/fixedsizelist.jl
View file
Original file line number Diff line number Diff line change
Expand Up @@ -169,16 +169,16 @@ function makenodesbuffers!(
len = length(col)
nc = nullcount(col)
push!(fieldnodes, FieldNode(len, nc))
@debugv 1 "made field node: nodeidx = $(length(fieldnodes)), col = $(typeof(col)), len = $(fieldnodes[end].length), nc = $(fieldnodes[end].null_count)"
@debug "made field node: nodeidx = $(length(fieldnodes)), col = $(typeof(col)), len = $(fieldnodes[end].length), nc = $(fieldnodes[end].null_count)"
# validity bitmap
blen = nc == 0 ? 0 : bitpackedbytes(len, alignment)
push!(fieldbuffers, Buffer(bufferoffset, blen))
@debugv 1 "made field buffer: bufferidx = $(length(fieldbuffers)), offset = $(fieldbuffers[end].offset), len = $(fieldbuffers[end].length), padded = $(padding(fieldbuffers[end].length, alignment))"
@debug "made field buffer: bufferidx = $(length(fieldbuffers)), offset = $(fieldbuffers[end].offset), len = $(fieldbuffers[end].length), padded = $(padding(fieldbuffers[end].length, alignment))"
bufferoffset += blen
if eltype(A) === UInt8
blen = ArrowTypes.getsize(ArrowTypes.ArrowKind(Base.nonmissingtype(T))) * len
push!(fieldbuffers, Buffer(bufferoffset, blen))
@debugv 1 "made field buffer: bufferidx = $(length(fieldbuffers)), offset = $(fieldbuffers[end].offset), len = $(fieldbuffers[end].length), padded = $(padding(fieldbuffers[end].length, alignment))"
@debug "made field buffer: bufferidx = $(length(fieldbuffers)), offset = $(fieldbuffers[end].offset), len = $(fieldbuffers[end].length), padded = $(padding(fieldbuffers[end].length, alignment))"
bufferoffset += padding(blen, alignment)
else
bufferoffset =
Expand All @@ -188,13 +188,13 @@ function makenodesbuffers!(
end

function writebuffer(io, col::FixedSizeList{T,A}, alignment) where {T,A}
@debugv 1 "writebuffer: col = $(typeof(col))"
@debugv 2 col
@debug "writebuffer: col = $(typeof(col))"
@debug col
writebitmap(io, col, alignment)
# write values array
if eltype(A) === UInt8
n = writearray(io, UInt8, col.data)
@debugv 1 "writing array: col = $(typeof(col.data)), n = $n, padded = $(padding(n, alignment))"
@debug "writing array: col = $(typeof(col.data)), n = $n, padded = $(padding(n, alignment))"
writezeros(io, paddinglength(n, alignment))
else
writebuffer(io, col.data, alignment)
Expand Down
16 changes: 8 additions & 8 deletions src/arraytypes/map.jl
View file
Original file line number Diff line number Diff line change
Expand Up @@ -103,21 +103,21 @@ function makenodesbuffers!(
len = length(col)
nc = nullcount(col)
push!(fieldnodes, FieldNode(len, nc))
@debugv 1 "made field node: nodeidx = $(length(fieldnodes)), col = $(typeof(col)), len = $(fieldnodes[end].length), nc = $(fieldnodes[end].null_count)"
@debug "made field node: nodeidx = $(length(fieldnodes)), col = $(typeof(col)), len = $(fieldnodes[end].length), nc = $(fieldnodes[end].null_count)"
# validity bitmap
blen = nc == 0 ? 0 : bitpackedbytes(len, alignment)
push!(fieldbuffers, Buffer(bufferoffset, blen))
@debugv 1 "made field buffer: bufferidx = $(length(fieldbuffers)), offset = $(fieldbuffers[end].offset), len = $(fieldbuffers[end].length), padded = $(padding(fieldbuffers[end].length, alignment))"
@debug "made field buffer: bufferidx = $(length(fieldbuffers)), offset = $(fieldbuffers[end].offset), len = $(fieldbuffers[end].length), padded = $(padding(fieldbuffers[end].length, alignment))"
# adjust buffer offset, make array buffer
bufferoffset += blen
blen = sizeof(O) * (len + 1)
push!(fieldbuffers, Buffer(bufferoffset, blen))
@debugv 1 "made field buffer: bufferidx = $(length(fieldbuffers)), offset = $(fieldbuffers[end].offset), len = $(fieldbuffers[end].length), padded = $(padding(fieldbuffers[end].length, alignment))"
@debug "made field buffer: bufferidx = $(length(fieldbuffers)), offset = $(fieldbuffers[end].offset), len = $(fieldbuffers[end].length), padded = $(padding(fieldbuffers[end].length, alignment))"
bufferoffset += padding(blen, alignment)
if liststringtype(col)
blen = length(col.data)
push!(fieldbuffers, Buffer(bufferoffset, blen))
@debugv 1 "made field buffer: bufferidx = $(length(fieldbuffers)), offset = $(fieldbuffers[end].offset), len = $(fieldbuffers[end].length), padded = $(padding(fieldbuffers[end].length, alignment))"
@debug "made field buffer: bufferidx = $(length(fieldbuffers)), offset = $(fieldbuffers[end].offset), len = $(fieldbuffers[end].length), padded = $(padding(fieldbuffers[end].length, alignment))"
bufferoffset += padding(blen, alignment)
else
bufferoffset =
Expand All @@ -127,17 +127,17 @@ function makenodesbuffers!(
end

function writebuffer(io, col::Union{Map{T,O,A},List{T,O,A}}, alignment) where {T,O,A}
@debugv 1 "writebuffer: col = $(typeof(col))"
@debugv 2 col
@debug "writebuffer: col = $(typeof(col))"
@debug col
writebitmap(io, col, alignment)
# write offsets
n = writearray(io, O, col.offsets.offsets)
@debugv 1 "writing array: col = $(typeof(col.offsets.offsets)), n = $n, padded = $(padding(n, alignment))"
@debug "writing array: col = $(typeof(col.offsets.offsets)), n = $n, padded = $(padding(n, alignment))"
writezeros(io, paddinglength(n, alignment))
# write values array
if liststringtype(col)
n = writearray(io, UInt8, col.data)
@debugv 1 "writing array: col = $(typeof(col.data)), n = $n, padded = $(padding(n, alignment))"
@debug "writing array: col = $(typeof(col.data)), n = $n, padded = $(padding(n, alignment))"
writezeros(io, paddinglength(n, alignment))
else
writebuffer(io, col.data, alignment)
Expand Down
12 changes: 6 additions & 6 deletions src/arraytypes/primitive.jl
View file
Original file line number Diff line number Diff line change
Expand Up @@ -88,25 +88,25 @@ function makenodesbuffers!(
len = length(col)
nc = nullcount(col)
push!(fieldnodes, FieldNode(len, nc))
@debugv 1 "made field node: nodeidx = $(length(fieldnodes)), col = $(typeof(col)), len = $(fieldnodes[end].length), nc = $(fieldnodes[end].null_count)"
@debug "made field node: nodeidx = $(length(fieldnodes)), col = $(typeof(col)), len = $(fieldnodes[end].length), nc = $(fieldnodes[end].null_count)"
# validity bitmap
blen = nc == 0 ? 0 : bitpackedbytes(len, alignment)
push!(fieldbuffers, Buffer(bufferoffset, blen))
@debugv 1 "made field buffer: bufferidx = $(length(fieldbuffers)), offset = $(fieldbuffers[end].offset), len = $(fieldbuffers[end].length), padded = $(padding(fieldbuffers[end].length, alignment))"
@debug "made field buffer: bufferidx = $(length(fieldbuffers)), offset = $(fieldbuffers[end].offset), len = $(fieldbuffers[end].length), padded = $(padding(fieldbuffers[end].length, alignment))"
# adjust buffer offset, make primitive array buffer
bufferoffset += blen
blen = len * sizeof(Base.nonmissingtype(T))
push!(fieldbuffers, Buffer(bufferoffset, blen))
@debugv 1 "made field buffer: bufferidx = $(length(fieldbuffers)), offset = $(fieldbuffers[end].offset), len = $(fieldbuffers[end].length), padded = $(padding(fieldbuffers[end].length, alignment))"
@debug "made field buffer: bufferidx = $(length(fieldbuffers)), offset = $(fieldbuffers[end].offset), len = $(fieldbuffers[end].length), padded = $(padding(fieldbuffers[end].length, alignment))"
return bufferoffset + padding(blen, alignment)
end

function writebuffer(io, col::Primitive{T}, alignment) where {T}
@debugv 1 "writebuffer: col = $(typeof(col))"
@debugv 2 col
@debug "writebuffer: col = $(typeof(col))"
@debug col
writebitmap(io, col, alignment)
n = writearray(io, Base.nonmissingtype(T), col.data)
@debugv 1 "writing array: col = $(typeof(col.data)), n = $n, padded = $(padding(n, alignment))"
@debug "writing array: col = $(typeof(col.data)), n = $n, padded = $(padding(n, alignment))"
writezeros(io, paddinglength(n, alignment))
return
end
8 changes: 4 additions & 4 deletions src/arraytypes/struct.jl
View file
Original file line number Diff line number Diff line change
Expand Up @@ -137,11 +137,11 @@ function makenodesbuffers!(
len = length(col)
nc = nullcount(col)
push!(fieldnodes, FieldNode(len, nc))
@debugv 1 "made field node: nodeidx = $(length(fieldnodes)), col = $(typeof(col)), len = $(fieldnodes[end].length), nc = $(fieldnodes[end].null_count)"
@debug "made field node: nodeidx = $(length(fieldnodes)), col = $(typeof(col)), len = $(fieldnodes[end].length), nc = $(fieldnodes[end].null_count)"
# validity bitmap
blen = nc == 0 ? 0 : bitpackedbytes(len, alignment)
push!(fieldbuffers, Buffer(bufferoffset, blen))
@debugv 1 "made field buffer: bufferidx = $(length(fieldbuffers)), offset = $(fieldbuffers[end].offset), len = $(fieldbuffers[end].length), padded = $(padding(fieldbuffers[end].length, alignment))"
@debug "made field buffer: bufferidx = $(length(fieldbuffers)), offset = $(fieldbuffers[end].offset), len = $(fieldbuffers[end].length), padded = $(padding(fieldbuffers[end].length, alignment))"
bufferoffset += blen
for child in col.data
bufferoffset =
Expand All @@ -151,8 +151,8 @@ function makenodesbuffers!(
end

function writebuffer(io, col::Struct, alignment)
@debugv 1 "writebuffer: col = $(typeof(col))"
@debugv 2 col
@debug "writebuffer: col = $(typeof(col))"
@debug col
writebitmap(io, col, alignment)
# write values arrays
for child in col.data
Expand Down
Loading

0 comments on commit aa5ad74

Please sign in to comment.