Skip to content

Read files directly #243

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 3 commits into from
Jan 4, 2023
Merged

Read files directly #243

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
efb0b0c
read files directly
onetonfoot Nov 15, 2022
8b7d2d5
add test cases
onetonfoot Nov 17, 2022
2ae5e08
remove stray lines
onetonfoot Nov 17, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
11 changes: 2 additions & 9 deletions src/gentypes.jl
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -437,10 +437,6 @@ function generate_struct_type_module(exprs, module_name)
return Expr(:module, true, module_name, type_block)
end

read_json_str(json_str) = read(
length(json_str) < 255 && isfile(json_str) ? Base.read(json_str, String) : json_str,
)

"""
JSON3.generatetypes(json, module_name; mutable=true, root_name=:Root)

Expand All @@ -460,9 +456,7 @@ function generatetypes(
root_name::Symbol = :Root,
)
# either a JSON.Array or JSON.Object
json = read_json_str(json_str)

# build a type for the JSON
json = read(json_str)
Copy link
Owner

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

wait, why are we not using read_json_str here anymore?

Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

read now does what read_json_str did, so it's redundant

Copy link
Owner

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

ah got it.

raw_json_type = generate_type(json)
json_exprs = generate_exprs(raw_json_type; root_name = root_name, mutable = mutable)
return generate_struct_type_module(json_exprs, module_name)
Expand All @@ -475,8 +469,7 @@ function generatetypes(
root_name::Symbol = :Root,
)
# either a JSON.Array or JSON.Object
json = read_json_str.(json_str)

json = read.(json_str)
# build a type for the JSON
raw_json_type = reduce(unify, type_or_eltype.(generate_type.(json)); init = Any)
json_exprs = generate_exprs(raw_json_type; root_name = root_name, mutable = mutable)
Expand Down
7 changes: 4 additions & 3 deletions src/read.jl
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,13 +10,13 @@ read(io::Union{IO, Base.AbstractCmd}; kw...) = read(Base.read(io, String); kw...
read(bytes::AbstractVector{UInt8}; kw...) = read(VectorString(bytes); kw...)

"""
JSON3.read(json_str, [type]; kw... )
JSON3.read(json, [type]; kw... )

Read JSON.

## Args

* `json_str`: A string, IO, or bytes (`AbstractVector{UInt8`) containing JSON to read
* `json`: A file, string, IO, or bytes (`AbstractVector{UInt8`) containing JSON to read
* `type`: Optionally, a type to read the JSON into. If not a [built in type](#Builtin-types), must have a "struct mapping" registered with [StructTypes.jl](#Struct-API).

## Keyword Args
Expand All @@ -27,9 +27,10 @@ Read JSON.
* `parsequoted`: Accept quoted values when reading into a NumberType. [default `false`]
* `numbertype`: Type to parse numbers as. [default `nothing`, which parses numbers as Int if possible, Float64 otherwise]
"""
function read(str::AbstractString; jsonlines::Bool=false,
function read(json::AbstractString; jsonlines::Bool=false,
numbertype::Union{DataType, Nothing}=nothing, kw...)

str = read_json_str(json)
buf = codeunits(str)
len = length(buf)
if len == 0
Expand Down
3 changes: 2 additions & 1 deletion src/structs.jl
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,8 @@ function rawbytes end
read(io::IO, ::Type{T}; kw...) where {T} = read(Base.read(io, String), T; kw...)
read(bytes::AbstractVector{UInt8}, ::Type{T}; kw...) where {T} = read(VectorString(bytes), T; kw...)

function _prepare_read(str::AbstractString, ::Type{T}) where {T}
function _prepare_read(json::AbstractString, ::Type{T}) where {T}
str = read_json_str(json)
buf = codeunits(str)
len = length(buf)
if len == 0
Expand Down
8 changes: 8 additions & 0 deletions src/utils.jl
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -188,3 +188,11 @@ Base.@pure function symbolin(names::Tuple{Vararg{Symbol}}, name::Symbol)
end
return false
end

function read_json_str(json)
# length check is to ensure that isfile doesn't thrown an error
# see issue for details https://github.com/JuliaLang/julia/issues/39774
!(json isa VectorString) && length(json) < 255 && isfile(json) ?
VectorString(Mmap.mmap(json)) :
json
end
9 changes: 9 additions & 0 deletions test/runtests.jl
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -398,6 +398,15 @@ x = XInt(10)
@test JSON3.read("{\"a\": 1, \"b\": 2}", NamedTuple{(:a, :b), Tuple{Float64, Float64}}) == (a=1.0, b=2.0)
@test JSON3.write((a=1.0, b=2.0)) == "{\"a\":1.0,\"b\":2.0}"

mktemp() do path, io
str = "{\"a\":1.0,\"b\":2.0}"
write(io, str)
obj = JSON3.read(str)
close(io)
@test JSON3.read(path) == obj
@test JSON3.read(path, Dict) == Dict("a" => 1.0, "b" => 2.0)
end

# Test that writing DictType works even when length(pairs(dict_type)) isn't
# available. Issue #37.
StructTypes.StructType(::Type{DictWithoutLength}) = StructTypes.DictType()
Expand Down